From 8cfbd4c287c2f18bb839ade9d0fe3e232cfe2249 Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 3 Aug 2026 15:45:36 +0800 Subject: [PATCH 01/71] perf: defer monitor availability detection --- .../common/constants/CommonConstants.java | 15 ++--- .../common/entity/manager/Monitor.java | 2 +- .../OldMonitorStatusWriteModelService.java | 4 +- .../service/impl/MonitorServiceImpl.java | 22 +++---- .../manager/service/MonitorServiceTest.java | 21 +++++++ ...DiscoveryExpansionSourceOwnershipTest.java | 2 +- ...OldMonitorStatusWriteModelServiceTest.java | 6 +- ...orStatusWriteModelSourceOwnershipTest.java | 4 +- .../warehouse/store/DataStorageDispatch.java | 11 ++-- .../store/DataStorageDispatchStatusTest.java | 60 +++++++++++++++++++ 10 files changed, 111 insertions(+), 36 deletions(-) create mode 100644 hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatchStatusTest.java diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/constants/CommonConstants.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/constants/CommonConstants.java index fe7455b1b5..16f6338c2c 100644 --- a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/constants/CommonConstants.java +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/constants/CommonConstants.java @@ -57,21 +57,18 @@ public interface CommonConstants { */ byte LOGIN_FAILED_CODE = 0x05; - /** - * Monitoring status 0: Paused, 1: Up, 2: Down - */ + /** Monitoring status 0: Paused. */ byte MONITOR_PAUSED_CODE = 0x00; - /** - * Monitoring status 0: Paused, 1: Up, 2: Down - */ + /** Monitoring status 1: Up. */ byte MONITOR_UP_CODE = 0x01; - /** - * Monitoring status 0: Paused, 1: Up, 2: Down - */ + /** Monitoring status 2: Down. */ byte MONITOR_DOWN_CODE = 0x02; + /** Monitoring status 3: Scheduled and waiting for the first availability result. */ + byte MONITOR_PENDING_CODE = 0x03; + /** * scrape type static */ diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/manager/Monitor.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/manager/Monitor.java index 07dd0f8578..2a6b9c344e 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/manager/Monitor.java +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/manager/Monitor.java @@ -98,7 +98,7 @@ public class Monitor { @Size(max = 100) private String cronExpression; - @Schema(title = "Task status 0: Paused, 1: Up, 2: Down", accessMode = READ_WRITE) + @Schema(title = "Task status 0: Paused, 1: Up, 2: Down, 3: Pending", accessMode = READ_WRITE) @Min(0) @Max(4) private byte status; diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/entity/OldMonitorStatusWriteModelService.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/entity/OldMonitorStatusWriteModelService.java index 48b9462a95..b3bdcf3700 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/entity/OldMonitorStatusWriteModelService.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/entity/OldMonitorStatusWriteModelService.java @@ -53,14 +53,14 @@ public class OldMonitorStatusWriteModelService { .collect(Collectors.toList()); } - public List findAndMarkPausedMonitorsUp(Set monitorIds) { + public List findAndMarkPausedMonitorsPending(Set monitorIds) { if (CollectionUtils.isEmpty(monitorIds)) { return List.of(); } return monitorDao.findMonitorsByIdIn(monitorIds) .stream() .filter(monitor -> monitor.getStatus() == CommonConstants.MONITOR_PAUSED_CODE) - .peek(monitor -> monitor.setStatus(CommonConstants.MONITOR_UP_CODE)) + .peek(monitor -> monitor.setStatus(CommonConstants.MONITOR_PENDING_CODE)) .collect(Collectors.toList()); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/MonitorServiceImpl.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/MonitorServiceImpl.java index 685e9227fa..eea211bec1 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/MonitorServiceImpl.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/MonitorServiceImpl.java @@ -221,14 +221,12 @@ public class MonitorServiceImpl implements MonitorService { return new Configmap(param.getField(), param.getParamValue(), param.getType()); }).collect(Collectors.toList()); appDefine.setConfigmap(configmaps); + // The cyclic job owns the first availability result. Persist a distinct + // pending state instead of blocking this write on a duplicate probe or + // presenting an unobserved target as healthy/down. + monitor.setStatus(CommonConstants.MONITOR_PENDING_CODE); long jobId = collector == null ? collectJobScheduling.addAsyncCollectJob(appDefine, null) : collectJobScheduling.addAsyncCollectJob(appDefine, collector); - try { - detectMonitor(monitor, params, collector); - } catch (Exception e) { - log.warn("Monitor detection failed during addMonitor for monitor [{}]: {}", - monitor.getName(), e.getMessage()); - } try { oldMonitorCollectorBindWriteModelService.saveCollectorBind(monitorId, collector); @@ -626,7 +624,7 @@ public class MonitorServiceImpl implements MonitorService { return; } List unManagedMonitors = oldMonitorStatusWriteModelService - .findAndMarkPausedMonitorsUp(allMonitorIds); + .findAndMarkPausedMonitorsPending(allMonitorIds); if (unManagedMonitors.isEmpty()) { return; } @@ -674,12 +672,6 @@ public class MonitorServiceImpl implements MonitorService { long newJobId = collectJobScheduling.addAsyncCollectJob(appDefine, collector); monitor.setJobId(newJobId); applicationContext.publishEvent(new MonitorDeletedEvent(applicationContext, monitor.getId())); - try { - detectMonitor(monitor, params, collector); - } catch (Exception e) { - log.warn("Monitor detection failed during reapplyMonitors for monitor [{}]: {}", - monitor.getName(), e.getMessage()); - } } oldMonitorStatusWriteModelService.saveMonitorStatusChanges(unManagedMonitors); } @@ -696,6 +688,9 @@ public class MonitorServiceImpl implements MonitorService { for (AppCount item : appCounts) { AppCount appCount = appCountMap.getOrDefault(item.getApp(), new AppCount()); appCount.setApp(item.getApp()); + // Preserve the total even for transitional or future statuses that + // do not belong to the three legacy availability counters. + appCount.setSize(appCount.getSize() + item.getSize()); switch (item.getStatus()) { case CommonConstants.MONITOR_UP_CODE -> appCount.setAvailableSize(appCount.getAvailableSize() + item.getSize()); @@ -711,7 +706,6 @@ public class MonitorServiceImpl implements MonitorService { // Traverse the map obtained by statistics and convert it into a List // result set return appCountMap.values().stream().map(item -> { - item.setSize(item.getAvailableSize() + item.getUnManageSize() + item.getUnAvailableSize()); try { Job job = appService.getAppDefine(item.getApp()); item.setCategory(job.getCategory()); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/MonitorServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/MonitorServiceTest.java index fae13d1725..64642e20f7 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/MonitorServiceTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/MonitorServiceTest.java @@ -25,6 +25,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.any; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.when; import java.util.ArrayList; @@ -328,6 +329,8 @@ class MonitorServiceTest { assertEquals(10, job.getDefaultInterval()); assertEquals("interval", job.getScheduleType()); assertNull(job.getCronExpression()); + assertEquals(CommonConstants.MONITOR_PENDING_CODE, monitor.getStatus()); + verify(collectJobScheduling, never()).collectSyncJobData(any(Job.class)); verify(entityIdentityResolutionService).refreshAutoMonitorBinds(monitor); } @@ -1240,6 +1243,7 @@ class MonitorServiceTest { assertEquals(10, job.getDefaultInterval()); assertEquals("cron", job.getScheduleType()); assertEquals("0 0 * * * ?", job.getCronExpression()); + monitors.forEach(monitor -> assertEquals(CommonConstants.MONITOR_PENDING_CODE, monitor.getStatus())); } @Test @@ -1278,6 +1282,23 @@ class MonitorServiceTest { assertDoesNotThrow(() -> monitorService.getAllAppMonitorsCount()); } + @Test + void getAllAppMonitorsCountIncludesPendingMonitorsInTotal() { + AppCount pending = new AppCount("test", CommonConstants.MONITOR_PENDING_CODE, 2L); + when(monitorDao.findAppsStatusCount()).thenReturn(List.of(pending)); + Job job = new Job(); + job.setMetrics(new ArrayList<>()); + when(appService.getAppDefine("test")).thenReturn(job); + + List result = monitorService.getAllAppMonitorsCount(); + + assertEquals(1, result.size()); + assertEquals(2L, result.getFirst().getSize()); + assertEquals(0L, result.getFirst().getAvailableSize()); + assertEquals(0L, result.getFirst().getUnAvailableSize()); + assertEquals(0L, result.getFirst().getUnManageSize()); + } + @Test void getMonitor() { long monitorId = 1L; diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/entity/OldMonitorServiceDiscoveryExpansionSourceOwnershipTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/entity/OldMonitorServiceDiscoveryExpansionSourceOwnershipTest.java index 84ac081d78..c6f410d19a 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/entity/OldMonitorServiceDiscoveryExpansionSourceOwnershipTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/entity/OldMonitorServiceDiscoveryExpansionSourceOwnershipTest.java @@ -60,7 +60,7 @@ class OldMonitorServiceDiscoveryExpansionSourceOwnershipTest { assertTrue(compactSource.indexOf("SetallMonitorIds=oldMonitorServiceDiscoveryExpansionService" + ".resolveMonitorIdsWithServiceDiscoveryChildren(ids);") < compactSource.indexOf("ListunManagedMonitors=oldMonitorStatusWriteModelService" - + ".findAndMarkPausedMonitorsUp(allMonitorIds);")); + + ".findAndMarkPausedMonitorsPending(allMonitorIds);")); assertTrue(Files.exists(OLD_MONITOR_SERVICE_DISCOVERY_EXPANSION_SERVICE)); String expansionSource = Files.readString(OLD_MONITOR_SERVICE_DISCOVERY_EXPANSION_SERVICE); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/entity/OldMonitorStatusWriteModelServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/entity/OldMonitorStatusWriteModelServiceTest.java index 0ec8b32a44..a45336405f 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/entity/OldMonitorStatusWriteModelServiceTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/entity/OldMonitorStatusWriteModelServiceTest.java @@ -69,17 +69,17 @@ class OldMonitorStatusWriteModelServiceTest { } @Test - void findAndMarkPausedMonitorsUpFiltersActiveRows() { + void findAndMarkPausedMonitorsPendingFiltersActiveRows() { Set monitorIds = Set.of(1L, 2L, 3L); Monitor upMonitor = Monitor.builder().id(1L).status(CommonConstants.MONITOR_UP_CODE).build(); Monitor pausedMonitor = Monitor.builder().id(2L).status(CommonConstants.MONITOR_PAUSED_CODE).build(); Monitor downMonitor = Monitor.builder().id(3L).status(CommonConstants.MONITOR_DOWN_CODE).build(); when(monitorDao.findMonitorsByIdIn(monitorIds)).thenReturn(List.of(upMonitor, pausedMonitor, downMonitor)); - List monitors = oldMonitorStatusWriteModelService.findAndMarkPausedMonitorsUp(monitorIds); + List monitors = oldMonitorStatusWriteModelService.findAndMarkPausedMonitorsPending(monitorIds); assertEquals(List.of(pausedMonitor), monitors); - assertEquals(CommonConstants.MONITOR_UP_CODE, pausedMonitor.getStatus()); + assertEquals(CommonConstants.MONITOR_PENDING_CODE, pausedMonitor.getStatus()); assertEquals(CommonConstants.MONITOR_UP_CODE, upMonitor.getStatus()); assertEquals(CommonConstants.MONITOR_DOWN_CODE, downMonitor.getStatus()); } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/entity/OldMonitorStatusWriteModelSourceOwnershipTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/entity/OldMonitorStatusWriteModelSourceOwnershipTest.java index 1363f4beb8..24c9daf518 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/entity/OldMonitorStatusWriteModelSourceOwnershipTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/entity/OldMonitorStatusWriteModelSourceOwnershipTest.java @@ -71,7 +71,7 @@ class OldMonitorStatusWriteModelSourceOwnershipTest { assertTrue(normalizedSource.contains( "oldMonitorStatusWriteModelService.findAndMarkManagedMonitorsPaused(allMonitorIds)")); assertTrue(normalizedSource.contains( - "oldMonitorStatusWriteModelService.findAndMarkPausedMonitorsUp(allMonitorIds)")); + "oldMonitorStatusWriteModelService.findAndMarkPausedMonitorsPending(allMonitorIds)")); assertTrue(normalizedSource.contains( "oldMonitorStatusWriteModelService.saveMonitorStatusChanges(managedMonitors)")); assertTrue(normalizedSource.contains( @@ -79,7 +79,7 @@ class OldMonitorStatusWriteModelSourceOwnershipTest { String writeModelSource = Files.readString(OLD_MONITOR_STATUS_WRITE_MODEL_SERVICE); assertTrue(writeModelSource.contains("public List findAndMarkManagedMonitorsPaused(Set")); - assertTrue(writeModelSource.contains("public List findAndMarkPausedMonitorsUp(Set")); + assertTrue(writeModelSource.contains("public List findAndMarkPausedMonitorsPending(Set")); assertTrue(writeModelSource.contains("public void saveMonitorStatusChanges(List monitors)")); assertTrue(writeModelSource.contains("monitorDao.findMonitorsByIdIn(monitorIds)")); assertTrue(writeModelSource.contains("monitorDao.saveAll(monitors)")); diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatch.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatch.java index 0684cdc8cd..9563e7ab4e 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatch.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatch.java @@ -159,10 +159,13 @@ public class DataStorageDispatch { long id = metricsData.getId(); CollectRep.Code code = metricsData.getCode(); try { - String sql = "UPDATE hzb_monitor SET status = ? WHERE id = ? AND status = ?"; - int status = code == CollectRep.Code.SUCCESS ? CommonConstants.MONITOR_UP_CODE : CommonConstants.MONITOR_DOWN_CODE; - int preStatus = code == CollectRep.Code.SUCCESS ? CommonConstants.MONITOR_DOWN_CODE : CommonConstants.MONITOR_UP_CODE; - int matchedRows = jdbcTemplate.update(sql, status, id, preStatus); + String sql = "UPDATE hzb_monitor SET status = ? WHERE id = ? AND status <> ? AND status <> ?"; + byte status = code == CollectRep.Code.SUCCESS + ? CommonConstants.MONITOR_UP_CODE + : CommonConstants.MONITOR_DOWN_CODE; + // Paused monitors must remain paused. Every other non-current + // state, including Pending, converges on the first priority-0 result. + int matchedRows = jdbcTemplate.update(sql, status, id, CommonConstants.MONITOR_PAUSED_CODE, status); if (matchedRows > 0) { entityManager.getEntityManagerFactory().getCache().evict(Monitor.class, id); } diff --git a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatchStatusTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatchStatusTest.java new file mode 100644 index 0000000000..2b85e5a637 --- /dev/null +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatchStatusTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.warehouse.store; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import java.util.List; +import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.queue.CommonDataQueue; +import org.apache.hertzbeat.plugin.runner.PluginRunner; +import org.apache.hertzbeat.warehouse.WarehouseWorkerPool; +import org.apache.hertzbeat.warehouse.store.realtime.RealTimeDataWriter; +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.core.JdbcTemplate; + +class DataStorageDispatchStatusTest { + + @Test + void firstAvailabilityResultCanReplacePendingStatus() { + JdbcTemplate jdbcTemplate = mock(JdbcTemplate.class); + DataStorageDispatch dispatch = new DataStorageDispatch( + mock(CommonDataQueue.class), + mock(WarehouseWorkerPool.class), + jdbcTemplate, + List.of(), + mock(RealTimeDataWriter.class), + mock(PluginRunner.class)); + CollectRep.MetricsData firstResult = CollectRep.MetricsData.newBuilder() + .setId(42L) + .setPriority(0) + .setCode(CollectRep.Code.SUCCESS) + .build(); + + dispatch.calculateMonitorStatus(firstResult); + + verify(jdbcTemplate).update( + "UPDATE hzb_monitor SET status = ? WHERE id = ? AND status <> ? AND status <> ?", + CommonConstants.MONITOR_UP_CODE, + 42L, + CommonConstants.MONITOR_PAUSED_CODE, + CommonConstants.MONITOR_UP_CODE); + } +} From 9e2e9572823945dc4efd16d3d3c1e29160012747 Mon Sep 17 00:00:00 2001 From: Logic Date: Thu, 6 Aug 2026 00:15:13 +0800 Subject: [PATCH 02/71] refactor: separate server and collector OTLP intake --- .../CollectorIntakeAdvertisementService.java | 3 + ...llectorIntakeAdvertisementServiceTest.java | 14 +- ...edInstrumentationIntakeProfileFactory.java | 100 ++++++++++++++ ...ExternalOtelCollectorIntakeProperties.java | 13 +- .../InstrumentationIntakeProperties.java | 31 +++++ ...agerInstrumentationIntakeProfileStore.java | 124 ++++++------------ ...ServerInstrumentationIntakeProperties.java | 19 +++ ...InstrumentationIntakeProfileStoreTest.java | 111 ++++++++++++---- ...erInstrumentationIntakePropertiesTest.java | 53 ++++++++ 9 files changed, 348 insertions(+), 120 deletions(-) create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/instrumentation/ConfiguredInstrumentationIntakeProfileFactory.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/instrumentation/InstrumentationIntakeProperties.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/instrumentation/ServerInstrumentationIntakeProperties.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/instrumentation/ServerInstrumentationIntakePropertiesTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/instrumentation/intake/CollectorIntakeAdvertisementService.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/instrumentation/intake/CollectorIntakeAdvertisementService.java index a48caa8a7e..aa43d6f1b6 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/instrumentation/intake/CollectorIntakeAdvertisementService.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/instrumentation/intake/CollectorIntakeAdvertisementService.java @@ -45,6 +45,9 @@ public class CollectorIntakeAdvertisementService implements CollectorIntakeAdver @Transactional(rollbackFor = Exception.class) public CollectorInstrumentationIntake update(String collectorName, CollectorIntakeAdvertisementRequest request) { + if (request.gateway() != Gateway.COLLECTOR) { + throw new IllegalArgumentException("Collector intake advertisement must be Collector-owned"); + } Collector collector = requireCollector(collectorName); collector.setInstrumentationIntake(codec.encode(request)); collectorDao.save(collector); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/instrumentation/intake/CollectorIntakeAdvertisementServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/instrumentation/intake/CollectorIntakeAdvertisementServiceTest.java index f9505e129e..ee0c398a13 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/instrumentation/intake/CollectorIntakeAdvertisementServiceTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/instrumentation/intake/CollectorIntakeAdvertisementServiceTest.java @@ -22,8 +22,11 @@ import static org.apache.hertzbeat.common.constants.CommonConstants.COLLECTOR_ST import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -53,7 +56,7 @@ class CollectorIntakeAdvertisementServiceTest { .status(COLLECTOR_STATUS_ONLINE) .build(); when(collectorDao.findCollectorByName("edge-west")).thenReturn(Optional.of(collector)); - CollectorIntakeAdvertisementRequest request = request(Gateway.SERVER); + CollectorIntakeAdvertisementRequest request = request(Gateway.COLLECTOR); CollectorIntakeAdvertisementService writer = service(collectorDao); CollectorInstrumentationIntake saved = writer.update("edge-west", request); @@ -72,6 +75,15 @@ class CollectorIntakeAdvertisementServiceTest { assertEquals(ErrorCode.INTAKE_NOT_ADVERTISED, cleared.errorCode()); } + @Test + void rejectsServerOwnedEndpointsAtTheCollectorMutationBoundary() { + CollectorDao collectorDao = mock(CollectorDao.class); + CollectorIntakeAdvertisementService service = service(collectorDao); + + assertThrows(IllegalArgumentException.class, () -> service.update("edge-west", request(Gateway.SERVER))); + verify(collectorDao, never()).save(any()); + } + @Test void mapsNotAdvertisedInvalidOfflineAndAvailableWithoutExposingStoredText(CapturedOutput output) { CollectorDao collectorDao = mock(CollectorDao.class); diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/instrumentation/ConfiguredInstrumentationIntakeProfileFactory.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/instrumentation/ConfiguredInstrumentationIntakeProfileFactory.java new file mode 100644 index 0000000000..449dab8449 --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/instrumentation/ConfiguredInstrumentationIntakeProfileFactory.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.instrumentation; + +import java.util.ArrayList; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.Authentication; +import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.Availability; +import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.ErrorCode; +import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.Gateway; +import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.IntakeEndpoint; +import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.IntakeKind; +import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.IntakeProfile; +import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.OtlpTransport; + +/** Maps explicit deployment configuration into a safe, secret-free destination profile. */ +final class ConfiguredInstrumentationIntakeProfileFactory { + + private static final String AUTHORIZATION_HEADER = "Authorization"; + private static final Pattern PROFILE_ID = Pattern.compile("[A-Za-z0-9][A-Za-z0-9._:-]{0,127}"); + + private ConfiguredInstrumentationIntakeProfileFactory() { + } + + static IntakeProfile create( + InstrumentationIntakeProperties properties, + IntakeKind kind, + Gateway gateway, + String safeInvalidId) { + String profileId = normalize(properties.profileId()); + if (profileId == null || !PROFILE_ID.matcher(profileId).matches()) { + return invalid(safeInvalidId, kind); + } + try { + EnumMap endpoints = new EnumMap<>(OtlpTransport.class); + List transports = new ArrayList<>(); + addEndpoint(transports, endpoints, OtlpTransport.HTTP_PROTOBUF, properties.otlpHttpEndpoint()); + addEndpoint(transports, endpoints, OtlpTransport.GRPC, properties.otlpGrpcEndpoint()); + if (transports.isEmpty()) { + return invalid(profileId, kind); + } + Authentication authentication = Authentication.fromCode(normalize(properties.authentication())); + return new IntakeProfile( + profileId, + kind, + Availability.AVAILABLE, + gateway, + transports, + endpoints, + authentication, + authentication == Authentication.BEARER_TOKEN ? AUTHORIZATION_HEADER : null, + null, + null); + } catch (IllegalArgumentException exception) { + // Deployment values can be sensitive even when malformed; expose only the stable contract code. + return invalid(profileId, kind); + } + } + + static IntakeProfile invalid(String profileId, IntakeKind kind) { + return new IntakeProfile( + profileId, + kind, + Availability.UNAVAILABLE, + null, + List.of(), + Map.of(), + null, + null, + ErrorCode.ADVERTISEMENT_INVALID); + } + + private static void addEndpoint( + List transports, + Map endpoints, + OtlpTransport transport, + String configuredEndpoint) { + String endpoint = normalize(configuredEndpoint); + if (endpoint != null) { + transports.add(transport); + endpoints.put(transport, IntakeEndpoint.fromUrl(endpoint)); + } + } + + private static String normalize(String value) { + if (value == null) { + return null; + } + String normalized = value.trim(); + return normalized.isEmpty() ? null : normalized; + } +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/instrumentation/ExternalOtelCollectorIntakeProperties.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/instrumentation/ExternalOtelCollectorIntakeProperties.java index 00614ed343..158872ebd4 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/instrumentation/ExternalOtelCollectorIntakeProperties.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/instrumentation/ExternalOtelCollectorIntakeProperties.java @@ -31,16 +31,5 @@ public record ExternalOtelCollectorIntakeProperties( String profileId, String otlpHttpEndpoint, String otlpGrpcEndpoint, - String authentication) { - - boolean configured() { - return hasText(profileId) - || hasText(otlpHttpEndpoint) - || hasText(otlpGrpcEndpoint) - || hasText(authentication); - } - - private boolean hasText(String value) { - return value != null && !value.isBlank(); - } + String authentication) implements InstrumentationIntakeProperties { } diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/instrumentation/InstrumentationIntakeProperties.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/instrumentation/InstrumentationIntakeProperties.java new file mode 100644 index 0000000000..3c74f9587e --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/instrumentation/InstrumentationIntakeProperties.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.instrumentation; + +/** Non-secret deployment properties shared by explicit OTLP destination types. */ +interface InstrumentationIntakeProperties { + + String profileId(); + + String otlpHttpEndpoint(); + + String otlpGrpcEndpoint(); + + String authentication(); + + default boolean configured() { + return hasText(profileId()) + || hasText(otlpHttpEndpoint()) + || hasText(otlpGrpcEndpoint()) + || hasText(authentication()); + } + + private static boolean hasText(String value) { + return value != null && !value.isBlank(); + } +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/instrumentation/ManagerInstrumentationIntakeProfileStore.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/instrumentation/ManagerInstrumentationIntakeProfileStore.java index c102d3d53d..8532fc8a69 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/instrumentation/ManagerInstrumentationIntakeProfileStore.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/instrumentation/ManagerInstrumentationIntakeProfileStore.java @@ -20,13 +20,11 @@ import java.util.ArrayList; import java.util.EnumMap; import java.util.List; import java.util.Map; -import java.util.regex.Pattern; import lombok.RequiredArgsConstructor; import org.apache.hertzbeat.manager.dao.CollectorDao; import org.apache.hertzbeat.manager.instrumentation.intake.CollectorIntakeAdvertisementReader; import org.apache.hertzbeat.manager.pojo.dto.CollectorInstrumentationIntake; import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.Availability; -import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.Authentication; import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.ErrorCode; import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.Gateway; import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.IntakeKind; @@ -46,21 +44,38 @@ import org.springframework.stereotype.Component; public class ManagerInstrumentationIntakeProfileStore implements InstrumentationIntakeProfileStore { private static final int MAX_PROFILES = 128; - private static final String AUTHORIZATION_HEADER = "Authorization"; + private static final String INVALID_SERVER_PROFILE_ID = "server:configured"; private static final String INVALID_EXTERNAL_PROFILE_ID = "external:configured"; - private static final Pattern PROFILE_ID = Pattern.compile("[A-Za-z0-9][A-Za-z0-9._:-]{0,127}"); private final CollectorDao collectorDao; private final CollectorIntakeAdvertisementReader advertisementReader; + private final ServerInstrumentationIntakeProperties serverProperties; private final ExternalOtelCollectorIntakeProperties externalProperties; @Override public List profiles() { - int managerProfileLimit = externalProperties.configured() ? MAX_PROFILES - 1 : MAX_PROFILES; - List profiles = new ArrayList<>(collectorDao - .findAll(PageRequest.of(0, managerProfileLimit, Sort.by("name").ascending())).stream() + int configuredProfiles = (serverProperties.configured() ? 1 : 0) + + (externalProperties.configured() ? 1 : 0); + int collectorProfileLimit = MAX_PROFILES - configuredProfiles; + List collectorProfiles = collectorDao + .findAll(PageRequest.of(0, collectorProfileLimit, Sort.by("name").ascending())).stream() .map(advertisementReader::read) + // Legacy Server advertisements remain readable on the Collector row for migration, + // but Server discovery is owned exclusively by the global deployment properties. + .filter(intake -> intake.gateway() != CollectorInstrumentationIntake.Gateway.SERVER) .map(this::map) - .toList()); + .toList(); + List profiles = new ArrayList<>(); + if (serverProperties.configured()) { + IntakeProfile configuredServer = mapServer(); + boolean profileIdCollides = collectorProfiles.stream() + .anyMatch(profile -> profile.id().equals(configuredServer.id())); + IntakeProfile server = profileIdCollides + ? ConfiguredInstrumentationIntakeProfileFactory.invalid( + INVALID_SERVER_PROFILE_ID, IntakeKind.SERVER) + : configuredServer; + profiles.add(server); + } + profiles.addAll(collectorProfiles); if (externalProperties.configured()) { IntakeProfile external = mapExternal(); String externalId = external.id(); @@ -73,21 +88,17 @@ public class ManagerInstrumentationIntakeProfileStore implements Instrumentation } private IntakeProfile map(CollectorInstrumentationIntake intake) { - IntakeKind kind = intake.gateway() == CollectorInstrumentationIntake.Gateway.SERVER - ? IntakeKind.SERVER - : IntakeKind.HERTZBEAT_COLLECTOR; - String id = (kind == IntakeKind.SERVER ? "server:" : "collector:") + intake.collectorId(); - String collectorId = kind == IntakeKind.HERTZBEAT_COLLECTOR ? intake.collectorId() : null; + String id = "collector:" + intake.collectorId(); if (intake.state() != CollectorInstrumentationIntake.State.AVAILABLE) { return new IntakeProfile( id, - kind, + IntakeKind.HERTZBEAT_COLLECTOR, Availability.UNAVAILABLE, null, List.of(), Map.of(), null, - collectorId, + intake.collectorId(), mapError(intake.errorCode())); } EnumMap endpoints = new EnumMap<>(OtlpTransport.class); @@ -102,87 +113,32 @@ public class ManagerInstrumentationIntakeProfileStore implements Instrumentation } return new IntakeProfile( id, - kind, + IntakeKind.HERTZBEAT_COLLECTOR, Availability.AVAILABLE, - kind == IntakeKind.SERVER ? Gateway.SERVER : Gateway.COLLECTOR, + Gateway.COLLECTOR, transports, endpoints, intake.authorizationHeader(), - collectorId, + intake.collectorId(), null); } - private IntakeProfile mapExternal() { - String profileId = normalize(externalProperties.profileId()); - if (profileId == null || !PROFILE_ID.matcher(profileId).matches()) { - return invalidExternal(INVALID_EXTERNAL_PROFILE_ID); - } - try { - EnumMap endpoints = new EnumMap<>(OtlpTransport.class); - List transports = new ArrayList<>(); - addExternalEndpoint( - transports, - endpoints, - OtlpTransport.HTTP_PROTOBUF, - externalProperties.otlpHttpEndpoint()); - addExternalEndpoint( - transports, - endpoints, - OtlpTransport.GRPC, - externalProperties.otlpGrpcEndpoint()); - if (transports.isEmpty()) { - return invalidExternal(profileId); - } - Authentication authentication = - Authentication.fromCode(normalize(externalProperties.authentication())); - return new IntakeProfile( - profileId, - IntakeKind.EXTERNAL_OTEL_COLLECTOR, - Availability.AVAILABLE, - Gateway.EXTERNAL, - transports, - endpoints, - authentication, - authentication == Authentication.BEARER_TOKEN ? AUTHORIZATION_HEADER : null, - null, - null); - } catch (IllegalArgumentException exception) { - // Deployment values can be sensitive even when malformed; expose only the stable contract code. - return invalidExternal(profileId); - } + private IntakeProfile mapServer() { + return ConfiguredInstrumentationIntakeProfileFactory.create( + serverProperties, IntakeKind.SERVER, Gateway.SERVER, INVALID_SERVER_PROFILE_ID); } - private void addExternalEndpoint( - List transports, - Map endpoints, - OtlpTransport transport, - String configuredEndpoint) { - String endpoint = normalize(configuredEndpoint); - if (endpoint != null) { - transports.add(transport); - endpoints.put(transport, IntakeEndpoint.fromUrl(endpoint)); - } + private IntakeProfile mapExternal() { + return ConfiguredInstrumentationIntakeProfileFactory.create( + externalProperties, + IntakeKind.EXTERNAL_OTEL_COLLECTOR, + Gateway.EXTERNAL, + INVALID_EXTERNAL_PROFILE_ID); } private IntakeProfile invalidExternal(String profileId) { - return new IntakeProfile( - profileId, - IntakeKind.EXTERNAL_OTEL_COLLECTOR, - Availability.UNAVAILABLE, - null, - List.of(), - Map.of(), - null, - null, - ErrorCode.ADVERTISEMENT_INVALID); - } - - private String normalize(String value) { - if (value == null) { - return null; - } - String normalized = value.trim(); - return normalized.isEmpty() ? null : normalized; + return ConfiguredInstrumentationIntakeProfileFactory.invalid( + profileId, IntakeKind.EXTERNAL_OTEL_COLLECTOR); } private ErrorCode mapError(CollectorInstrumentationIntake.ErrorCode errorCode) { diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/instrumentation/ServerInstrumentationIntakeProperties.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/instrumentation/ServerInstrumentationIntakeProperties.java new file mode 100644 index 0000000000..f417b63bdc --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/instrumentation/ServerInstrumentationIntakeProperties.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.instrumentation; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** Explicit public OTLP endpoints owned by HertzBeat Server, never by a Collector row. */ +@ConfigurationProperties(prefix = "hertzbeat.instrumentation.server") +public record ServerInstrumentationIntakeProperties( + String profileId, + String otlpHttpEndpoint, + String otlpGrpcEndpoint, + String authentication) implements InstrumentationIntakeProperties { +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/instrumentation/ManagerInstrumentationIntakeProfileStoreTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/instrumentation/ManagerInstrumentationIntakeProfileStoreTest.java index ebd818a30d..b08e41c89d 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/instrumentation/ManagerInstrumentationIntakeProfileStoreTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/instrumentation/ManagerInstrumentationIntakeProfileStoreTest.java @@ -44,6 +44,7 @@ import org.apache.hertzbeat.observability.instrumentation.v2.api.Instrumentation import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.ErrorCode; import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.Gateway; import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.IntakeKind; +import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.IntakeProfile; import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.OtlpTransport; import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.TransportSecurity; import org.apache.hertzbeat.observability.instrumentation.v2.service.InstrumentationApplicationGuideV2Adapter; @@ -57,7 +58,7 @@ import org.springframework.data.domain.Pageable; class ManagerInstrumentationIntakeProfileStoreTest { @Test - void mapsOnlyExistingExplicitAdvertisementsWithoutInferringEndpoints() { + void mapsGlobalServerAndCollectorDestinationsWithoutUsingLegacyServerAdvertisements() { Collector server = collector("server-advertisement"); Collector loopback = collector("loopback"); Collector edge = collector("edge"); @@ -70,29 +71,69 @@ class ManagerInstrumentationIntakeProfileStoreTest { "edge", CollectorInstrumentationIntake.ErrorCode.INTAKE_ADVERTISEMENT_UNAVAILABLE)); var profiles = new ManagerInstrumentationIntakeProfileStore( - dao, reader, unconfiguredExternal()).profiles(); + dao, reader, configuredServer(), unconfiguredExternal()).profiles(); assertEquals(3, profiles.size()); - assertEquals("server:server-advertisement", profiles.getFirst().id()); - assertEquals(IntakeKind.SERVER, profiles.getFirst().kind()); - assertEquals("https://otel.example.test/v1", profiles.getFirst() + IntakeProfile serverProfile = profile(profiles, "server-direct"); + assertEquals(IntakeKind.SERVER, serverProfile.kind()); + assertEquals("https://server.example.test/api/otlp", serverProfile .endpoints().get(OtlpTransport.HTTP_PROTOBUF).url()); - assertEquals(Authentication.BEARER_TOKEN, profiles.getFirst().authentication()); - assertEquals("Authorization", profiles.getFirst().authorizationHeader()); - assertEquals(TransportSecurity.TLS, profiles.getFirst() + assertEquals(Authentication.BEARER_TOKEN, serverProfile.authentication()); + assertEquals("Authorization", serverProfile.authorizationHeader()); + assertEquals(TransportSecurity.TLS, serverProfile .endpoints().get(OtlpTransport.HTTP_PROTOBUF).security()); - assertNull(profiles.getFirst().collectorId()); - assertEquals(IntakeKind.HERTZBEAT_COLLECTOR, profiles.get(1).kind()); - assertEquals(Availability.AVAILABLE, profiles.get(1).availability()); - assertEquals("loopback", profiles.get(1).collectorId()); - assertEquals("http://127.0.0.1:4318", profiles.get(1) + assertNull(serverProfile.collectorId()); + + IntakeProfile collectorProfile = profile(profiles, "collector:loopback"); + assertEquals(IntakeKind.HERTZBEAT_COLLECTOR, collectorProfile.kind()); + assertEquals(Availability.AVAILABLE, collectorProfile.availability()); + assertEquals("loopback", collectorProfile.collectorId()); + assertEquals("http://127.0.0.1:4318", collectorProfile .endpoints().get(OtlpTransport.HTTP_PROTOBUF).url()); - assertEquals(TransportSecurity.PLAINTEXT, profiles.get(1) + assertEquals(TransportSecurity.PLAINTEXT, collectorProfile .endpoints().get(OtlpTransport.HTTP_PROTOBUF).security()); - assertEquals(IntakeKind.HERTZBEAT_COLLECTOR, profiles.get(2).kind()); - assertEquals(Availability.UNAVAILABLE, profiles.get(2).availability()); - assertEquals("edge", profiles.get(2).collectorId()); - assertEquals(true, profiles.get(2).endpoints().isEmpty()); + + IntakeProfile unavailableCollector = profile(profiles, "collector:edge"); + assertEquals(IntakeKind.HERTZBEAT_COLLECTOR, unavailableCollector.kind()); + assertEquals(Availability.UNAVAILABLE, unavailableCollector.availability()); + assertEquals("edge", unavailableCollector.collectorId()); + assertTrue(unavailableCollector.endpoints().isEmpty()); + assertTrue(profiles.stream().noneMatch(profile -> profile.id().equals("server:server-advertisement"))); + } + + @Test + void legacyCollectorServerAdvertisementDoesNotCreateAnImplicitServerDestination() { + Collector server = collector("server-advertisement"); + CollectorDao dao = mock(CollectorDao.class); + CollectorIntakeAdvertisementReader reader = mock(CollectorIntakeAdvertisementReader.class); + when(dao.findAll(any(Pageable.class))).thenReturn(new PageImpl<>(List.of(server))); + when(reader.read(server)).thenReturn(availableServer()); + + var profiles = new ManagerInstrumentationIntakeProfileStore( + dao, reader, unconfiguredServer(), unconfiguredExternal()).profiles(); + + assertTrue(profiles.isEmpty()); + } + + @Test + void serverProfileIdCollisionCannotCreateAnAmbiguousCollectorDestination() { + Collector loopback = collector("loopback"); + CollectorDao dao = mock(CollectorDao.class); + CollectorIntakeAdvertisementReader reader = mock(CollectorIntakeAdvertisementReader.class); + when(dao.findAll(any(Pageable.class))).thenReturn(new PageImpl<>(List.of(loopback))); + when(reader.read(loopback)).thenReturn(availableLoopback()); + ServerInstrumentationIntakeProperties collidingServer = new ServerInstrumentationIntakeProperties( + "collector:loopback", "https://server.example.test/api/otlp", null, "bearer_token"); + + var profiles = new ManagerInstrumentationIntakeProfileStore( + dao, reader, collidingServer, unconfiguredExternal()).profiles(); + + assertEquals(2, profiles.size()); + IntakeProfile serverProfile = profile(profiles, "server:configured"); + assertEquals(IntakeKind.SERVER, serverProfile.kind()); + assertEquals(Availability.UNAVAILABLE, serverProfile.availability()); + assertEquals(ErrorCode.ADVERTISEMENT_INVALID, serverProfile.errorCode()); + assertEquals(Availability.AVAILABLE, profile(profiles, "collector:loopback").availability()); } @Test @@ -105,6 +146,7 @@ class ManagerInstrumentationIntakeProfileStoreTest { var store = new ManagerInstrumentationIntakeProfileStore( dao, reader, + configuredServer(), new ExternalOtelCollectorIntakeProperties( "external-west", "http://otel.example.test:4318", @@ -114,7 +156,7 @@ class ManagerInstrumentationIntakeProfileStoreTest { var discovery = profiles.profiles(); assertEquals(2, discovery.profiles().size()); - assertEquals("server:server-advertisement", discovery.defaultProfileId()); + assertEquals("server-direct", discovery.defaultProfileId()); var external = discovery.profiles().stream() .filter(profile -> profile.id().equals("external-west")) .findFirst() @@ -171,6 +213,7 @@ class ManagerInstrumentationIntakeProfileStoreTest { var profile = new ManagerInstrumentationIntakeProfileStore( emptyDao(), mock(CollectorIntakeAdvertisementReader.class), + unconfiguredServer(), new ExternalOtelCollectorIntakeProperties( "external-west", endpoint, null, "bearer_token")) .profiles() @@ -194,6 +237,7 @@ class ManagerInstrumentationIntakeProfileStoreTest { var none = new ManagerInstrumentationIntakeProfileStore( emptyDao(), mock(CollectorIntakeAdvertisementReader.class), + unconfiguredServer(), new ExternalOtelCollectorIntakeProperties( "external-none", "http://otel.example.test:4318", @@ -217,6 +261,7 @@ class ManagerInstrumentationIntakeProfileStoreTest { var invalid = new ManagerInstrumentationIntakeProfileStore( emptyDao(), mock(CollectorIntakeAdvertisementReader.class), + unconfiguredServer(), new ExternalOtelCollectorIntakeProperties( "external-invalid", "https://otel.example.test:4318", @@ -365,13 +410,15 @@ class ManagerInstrumentationIntakeProfileStoreTest { @Test void absentConfigurationCreatesNoProfileWhileIncompleteOrUnsafeIdUsesSafeFailureId() { assertTrue(new ManagerInstrumentationIntakeProfileStore( - emptyDao(), mock(CollectorIntakeAdvertisementReader.class), unconfiguredExternal()) + emptyDao(), mock(CollectorIntakeAdvertisementReader.class), + unconfiguredServer(), unconfiguredExternal()) .profiles() .isEmpty()); var incomplete = new ManagerInstrumentationIntakeProfileStore( emptyDao(), mock(CollectorIntakeAdvertisementReader.class), + unconfiguredServer(), new ExternalOtelCollectorIntakeProperties( "external-west", null, null, "bearer_token")) .profiles() @@ -382,6 +429,7 @@ class ManagerInstrumentationIntakeProfileStoreTest { var unsafeId = new ManagerInstrumentationIntakeProfileStore( emptyDao(), mock(CollectorIntakeAdvertisementReader.class), + unconfiguredServer(), new ExternalOtelCollectorIntakeProperties( "external?token=secret-value", "https://otel.example.test:4318", @@ -404,8 +452,9 @@ class ManagerInstrumentationIntakeProfileStoreTest { var store = new ManagerInstrumentationIntakeProfileStore( dao, reader, + configuredServer(), new ExternalOtelCollectorIntakeProperties( - "server:server-advertisement", + "server-direct", "https://otel.example.test:4318", null, "bearer_token")); @@ -413,7 +462,7 @@ class ManagerInstrumentationIntakeProfileStoreTest { var discovery = new InstrumentationIntakeProfileV2Service(store).profiles(); assertEquals(2, discovery.profiles().size()); - assertEquals("server:server-advertisement", discovery.defaultProfileId()); + assertEquals("server-direct", discovery.defaultProfileId()); assertEquals("external:configured", discovery.profiles().get(1).id()); assertEquals(Availability.UNAVAILABLE, discovery.profiles().get(1).availability()); assertEquals(ErrorCode.ADVERTISEMENT_INVALID, discovery.profiles().get(1).errorCode()); @@ -435,12 +484,28 @@ class ManagerInstrumentationIntakeProfileStoreTest { return new ExternalOtelCollectorIntakeProperties(null, null, null, null); } + private ServerInstrumentationIntakeProperties unconfiguredServer() { + return new ServerInstrumentationIntakeProperties(null, null, null, null); + } + + private ServerInstrumentationIntakeProperties configuredServer() { + return new ServerInstrumentationIntakeProperties( + "server-direct", "https://server.example.test/api/otlp", null, "bearer_token"); + } + + private IntakeProfile profile(List profiles, String profileId) { + return profiles.stream() + .filter(profile -> profile.id().equals(profileId)) + .findFirst() + .orElseThrow(); + } + private InstrumentationGuideV2Renderer renderer(ExternalOtelCollectorIntakeProperties properties) { InstrumentationCatalogV2Service catalog = new InstrumentationCatalogV2Service(new InstrumentationCatalogService()); InstrumentationIntakeProfileV2Service profiles = new InstrumentationIntakeProfileV2Service( new ManagerInstrumentationIntakeProfileStore( - emptyDao(), mock(CollectorIntakeAdvertisementReader.class), properties)); + emptyDao(), mock(CollectorIntakeAdvertisementReader.class), unconfiguredServer(), properties)); return new InstrumentationGuideV2Renderer( catalog, profiles, diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/instrumentation/ServerInstrumentationIntakePropertiesTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/instrumentation/ServerInstrumentationIntakePropertiesTest.java new file mode 100644 index 0000000000..11b0de9c60 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/instrumentation/ServerInstrumentationIntakePropertiesTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.instrumentation; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Arrays; +import org.junit.jupiter.api.Test; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; + +class ServerInstrumentationIntakePropertiesTest { + + private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() + .withUserConfiguration(BindingConfig.class); + + @Test + void bindsOnlyExplicitGlobalNonSecretServerProfileFields() { + contextRunner.withPropertyValues( + "hertzbeat.instrumentation.server.profile-id=server-direct", + "hertzbeat.instrumentation.server.otlp-http-endpoint=http://server.example.test:1157/api/otlp", + "hertzbeat.instrumentation.server.otlp-grpc-endpoint=http://server.example.test:4317", + "hertzbeat.instrumentation.server.authentication=bearer_token") + .run(context -> { + ServerInstrumentationIntakeProperties properties = + context.getBean(ServerInstrumentationIntakeProperties.class); + assertEquals("server-direct", properties.profileId()); + assertEquals("http://server.example.test:1157/api/otlp", properties.otlpHttpEndpoint()); + assertEquals("http://server.example.test:4317", properties.otlpGrpcEndpoint()); + assertEquals("bearer_token", properties.authentication()); + }); + + assertThat(Arrays.stream(ServerInstrumentationIntakeProperties.class.getRecordComponents()) + .map(component -> component.getName())) + .containsExactly("profileId", "otlpHttpEndpoint", "otlpGrpcEndpoint", "authentication"); + } + + @Test + void remainsUnconfiguredWithoutAnExplicitPublicEndpoint() { + contextRunner.run(context -> assertThat(context.getBean(ServerInstrumentationIntakeProperties.class) + .configured()).isFalse()); + } + + @EnableConfigurationProperties(ServerInstrumentationIntakeProperties.class) + static class BindingConfig { + } +} From 08960234e4a5814da5956c637e76d145805888ec Mon Sep 17 00:00:00 2001 From: Logic Date: Thu, 6 Aug 2026 00:27:28 +0800 Subject: [PATCH 03/71] feat: verify collector OTLP gateway runtime --- .../otel/OtelRuntimeStatusProvider.java | 18 ++++- .../otel/OtelRuntimeStatusProviderTest.java | 36 +++++++++ .../entity/dto/ManagedOtelRuntimeStatus.java | 80 ++++++++++++++++++- .../dto/ManagedOtelRuntimeStatusTest.java | 33 ++++++++ .../CollectorIntakeAdvertisementService.java | 39 ++++++++- ...llectorIntakeAdvertisementServiceTest.java | 78 +++++++++++++++--- 6 files changed, 266 insertions(+), 18 deletions(-) diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/runtime/otel/OtelRuntimeStatusProvider.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/runtime/otel/OtelRuntimeStatusProvider.java index 392ddac8bc..e913150668 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/runtime/otel/OtelRuntimeStatusProvider.java +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/runtime/otel/OtelRuntimeStatusProvider.java @@ -22,6 +22,8 @@ import org.apache.hertzbeat.collector.dispatch.CollectorRuntimeStatusProvider; import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus; import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus.FailureCode; import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus.ObservedLong; +import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus.OtlpGatewayStatus; +import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus.OtlpGatewayTransport; import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus.RuntimeTelemetry; /** @@ -84,10 +86,24 @@ public class OtelRuntimeStatusProvider implements CollectorRuntimeStatusProvider diagnosticsReader.sanitize(snapshot.lastError(), properties), failureCode, telemetry, - sources + sources, + otlpGateway(snapshot) ); } + private OtlpGatewayStatus otlpGateway(OtelRuntimeSnapshot snapshot) { + if (!properties.isOtlpGatewayEnabled()) { + return OtlpGatewayStatus.disabled(); + } + if (!properties.isEnabled() || snapshot.state() != OtelRuntimeState.RUNNING) { + return OtlpGatewayStatus.unavailable(); + } + // Gateway mode renders both OTLP receivers into the same health-checked runtime. + return OtlpGatewayStatus.available(List.of( + OtlpGatewayTransport.HTTP_PROTOBUF, + OtlpGatewayTransport.GRPC)); + } + private List sanitize( List sources) { return sources.stream() diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/runtime/otel/OtelRuntimeStatusProviderTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/runtime/otel/OtelRuntimeStatusProviderTest.java index fe320b06c3..59bdf02ca0 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/runtime/otel/OtelRuntimeStatusProviderTest.java +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/runtime/otel/OtelRuntimeStatusProviderTest.java @@ -41,6 +41,7 @@ class OtelRuntimeStatusProviderTest { properties.setEnabled(true); properties.setConfigRevision(12); properties.setToken("managed-intake-token"); + properties.setOtlpGatewayEnabled(true); OtelRuntimeSupervisor supervisor = mock(OtelRuntimeSupervisor.class); when(supervisor.snapshot()).thenReturn(new OtelRuntimeSnapshot( OtelRuntimeState.RUNNING, 42, 2, Instant.parse("2026-07-15T06:00:00Z"), "")); @@ -72,6 +73,12 @@ class OtelRuntimeStatusProviderTest { status.intakeCredentialState()); assertEquals(List.of(source), status.sources()); assertEquals(telemetry, status.telemetry()); + assertEquals(ManagedOtelRuntimeStatus.OtlpGatewayState.AVAILABLE, + status.otlpGateway().state()); + assertEquals(List.of( + ManagedOtelRuntimeStatus.OtlpGatewayTransport.HTTP_PROTOBUF, + ManagedOtelRuntimeStatus.OtlpGatewayTransport.GRPC), + status.otlpGateway().supportedTransports()); } @Test @@ -93,6 +100,35 @@ class OtelRuntimeStatusProviderTest { assertEquals(ManagedOtelRuntimeStatus.IntakeCredentialState.NOT_REQUIRED, status.intakeCredentialState()); + assertEquals(ManagedOtelRuntimeStatus.OtlpGatewayState.DISABLED, + status.otlpGateway().state()); + } + + @Test + void enabledGatewayIsUnavailableUntilTheManagedRuntimeIsRunning() { + OtelRuntimeProperties properties = new OtelRuntimeProperties(); + properties.setEnabled(true); + properties.setOtlpGatewayEnabled(true); + properties.setToken("managed-intake-token"); + OtelRuntimeSupervisor supervisor = mock(OtelRuntimeSupervisor.class); + when(supervisor.snapshot()).thenReturn(new OtelRuntimeSnapshot( + OtelRuntimeState.STARTING, -1, 0, Instant.parse("2026-07-15T06:00:00Z"), "")); + when(supervisor.sourceStatuses()).thenReturn(List.of()); + OtelRuntimeDiagnosticsReader diagnosticsReader = mock(OtelRuntimeDiagnosticsReader.class); + when(diagnosticsReader.latestFailure(properties)).thenReturn(ManagedOtelRuntimeStatus.FailureCode.NONE); + when(diagnosticsReader.sanitize("", properties)).thenReturn(""); + OtelRuntimeStatusProvider provider = new OtelRuntimeStatusProvider( + properties, + supervisor, + mock(OtelRuntimeTelemetryClient.class), + diagnosticsReader, + new OtelRuntimeFailureClassifier()); + + ManagedOtelRuntimeStatus status = provider.status(); + + assertEquals(ManagedOtelRuntimeStatus.OtlpGatewayState.UNAVAILABLE, + status.otlpGateway().state()); + assertTrue(status.otlpGateway().supportedTransports().isEmpty()); } @Test diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/dto/ManagedOtelRuntimeStatus.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/dto/ManagedOtelRuntimeStatus.java index 73af9a262d..fae46769c4 100644 --- a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/dto/ManagedOtelRuntimeStatus.java +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/dto/ManagedOtelRuntimeStatus.java @@ -18,6 +18,7 @@ package org.apache.hertzbeat.common.entity.dto; import java.time.Instant; +import java.util.Comparator; import java.util.List; import java.util.Objects; import java.util.regex.Pattern; @@ -34,10 +35,12 @@ public record ManagedOtelRuntimeStatus(int schemaVersion, boolean enabled, Runti IntakeCredentialState intakeCredentialState, int restartCount, Instant changedAt, String lastError, FailureCode failureCode, RuntimeTelemetry telemetry, - List sources) { + List sources, + OtlpGatewayStatus otlpGateway) { - public static final int CURRENT_SCHEMA_VERSION = 2; + public static final int CURRENT_SCHEMA_VERSION = 3; private static final int LEGACY_SCHEMA_VERSION = 1; + private static final int OTLP_GATEWAY_SCHEMA_VERSION = 3; private static final int MAXIMUM_DIAGNOSTIC_LENGTH = 512; // Active sources plus both sides of one pending/rejected replacement revision. private static final int MAXIMUM_SOURCE_STATUSES = 147; @@ -49,7 +52,7 @@ public record ManagedOtelRuntimeStatus(int schemaVersion, boolean enabled, Runti int restartCount, Instant changedAt, String lastError) { this(schemaVersion, enabled, state, desiredRevision, activeRevision, -1, intakeCredentialState, restartCount, changedAt, lastError, FailureCode.NONE, - RuntimeTelemetry.unavailable(false), List.of()); + RuntimeTelemetry.unavailable(false), List.of(), OtlpGatewayStatus.notReported()); } public ManagedOtelRuntimeStatus(int schemaVersion, boolean enabled, RuntimeState state, @@ -59,7 +62,20 @@ public record ManagedOtelRuntimeStatus(int schemaVersion, boolean enabled, Runti List sources) { this(schemaVersion, enabled, state, desiredRevision, activeRevision, -1, intakeCredentialState, restartCount, changedAt, lastError, FailureCode.NONE, - RuntimeTelemetry.unavailable(hasFileSource(sources)), sources); + RuntimeTelemetry.unavailable(hasFileSource(sources)), sources, + OtlpGatewayStatus.notReported()); + } + + /** Preserves the schema 1/2 constructor shape while gateway reporting rolls out. */ + public ManagedOtelRuntimeStatus(int schemaVersion, boolean enabled, RuntimeState state, + long desiredRevision, long activeRevision, long pid, + IntakeCredentialState intakeCredentialState, + int restartCount, Instant changedAt, String lastError, + FailureCode failureCode, RuntimeTelemetry telemetry, + List sources) { + this(schemaVersion, enabled, state, desiredRevision, activeRevision, pid, + intakeCredentialState, restartCount, changedAt, lastError, failureCode, + telemetry, sources, OtlpGatewayStatus.notReported()); } public ManagedOtelRuntimeStatus { @@ -91,6 +107,11 @@ public record ManagedOtelRuntimeStatus(int schemaVersion, boolean enabled, Runti } failureCode = Objects.requireNonNullElse(failureCode, FailureCode.NONE); telemetry = telemetry == null ? RuntimeTelemetry.unavailable(hasFileSource(sources)) : telemetry; + otlpGateway = otlpGateway == null ? OtlpGatewayStatus.notReported() : otlpGateway; + if (schemaVersion < OTLP_GATEWAY_SCHEMA_VERSION + && otlpGateway.state() != OtlpGatewayState.NOT_REPORTED) { + throw new IllegalArgumentException("OTLP Gateway status requires managed runtime status schema 3"); + } } private static boolean hasFileSource(List sources) { @@ -134,6 +155,57 @@ public record ManagedOtelRuntimeStatus(int schemaVersion, boolean enabled, Runti UNKNOWN } + /** Lifecycle of the explicitly enabled application-telemetry Gateway listener. */ + public enum OtlpGatewayState { + NOT_REPORTED, + DISABLED, + UNAVAILABLE, + AVAILABLE + } + + /** Payload-free OTLP protocols confirmed by the running managed runtime. */ + public enum OtlpGatewayTransport { + HTTP_PROTOBUF, + GRPC + } + + /** Bounded runtime proof used to validate separately advertised public endpoints. */ + public record OtlpGatewayStatus(OtlpGatewayState state, + List supportedTransports) { + + public OtlpGatewayStatus { + state = Objects.requireNonNull(state, "state"); + supportedTransports = supportedTransports == null + ? List.of() + : supportedTransports.stream() + .distinct() + .sorted(Comparator.comparingInt(OtlpGatewayTransport::ordinal)) + .toList(); + if (state == OtlpGatewayState.AVAILABLE && supportedTransports.isEmpty()) { + throw new IllegalArgumentException("Available OTLP Gateway must report a transport"); + } + if (state != OtlpGatewayState.AVAILABLE && !supportedTransports.isEmpty()) { + throw new IllegalArgumentException("Unavailable OTLP Gateway cannot report transports"); + } + } + + public static OtlpGatewayStatus notReported() { + return new OtlpGatewayStatus(OtlpGatewayState.NOT_REPORTED, List.of()); + } + + public static OtlpGatewayStatus disabled() { + return new OtlpGatewayStatus(OtlpGatewayState.DISABLED, List.of()); + } + + public static OtlpGatewayStatus unavailable() { + return new OtlpGatewayStatus(OtlpGatewayState.UNAVAILABLE, List.of()); + } + + public static OtlpGatewayStatus available(List transports) { + return new OtlpGatewayStatus(OtlpGatewayState.AVAILABLE, transports); + } + } + /** * Availability of one numeric runtime metric. Zero is meaningful only when available. */ diff --git a/hertzbeat-common-core/src/test/java/org/apache/hertzbeat/common/entity/dto/ManagedOtelRuntimeStatusTest.java b/hertzbeat-common-core/src/test/java/org/apache/hertzbeat/common/entity/dto/ManagedOtelRuntimeStatusTest.java index e397090c42..944041d11b 100644 --- a/hertzbeat-common-core/src/test/java/org/apache/hertzbeat/common/entity/dto/ManagedOtelRuntimeStatusTest.java +++ b/hertzbeat-common-core/src/test/java/org/apache/hertzbeat/common/entity/dto/ManagedOtelRuntimeStatusTest.java @@ -190,5 +190,38 @@ class ManagedOtelRuntimeStatusTest { status.telemetry().queueSize().state()); assertEquals(ManagedOtelRuntimeStatus.ValueState.UNAVAILABLE, status.telemetry().queueCapacityBySignal().traces().state()); + assertEquals(ManagedOtelRuntimeStatus.OtlpGatewayState.NOT_REPORTED, + status.otlpGateway().state()); + } + + @Test + void carriesOnlyBoundedPayloadFreeOtlpGatewayCapabilities() { + ManagedOtelRuntimeStatus.OtlpGatewayStatus gateway = + ManagedOtelRuntimeStatus.OtlpGatewayStatus.available(List.of( + ManagedOtelRuntimeStatus.OtlpGatewayTransport.HTTP_PROTOBUF, + ManagedOtelRuntimeStatus.OtlpGatewayTransport.GRPC)); + ManagedOtelRuntimeStatus status = new ManagedOtelRuntimeStatus( + ManagedOtelRuntimeStatus.CURRENT_SCHEMA_VERSION, + true, + ManagedOtelRuntimeStatus.RuntimeState.RUNNING, + 1, + 1, + -1, + ManagedOtelRuntimeStatus.IntakeCredentialState.CONFIGURED, + 0, + Instant.now(), + "", + ManagedOtelRuntimeStatus.FailureCode.NONE, + ManagedOtelRuntimeStatus.RuntimeTelemetry.unavailable(false), + List.of(), + gateway); + + assertEquals(ManagedOtelRuntimeStatus.OtlpGatewayState.AVAILABLE, + status.otlpGateway().state()); + assertEquals(List.of( + ManagedOtelRuntimeStatus.OtlpGatewayTransport.HTTP_PROTOBUF, + ManagedOtelRuntimeStatus.OtlpGatewayTransport.GRPC), + status.otlpGateway().supportedTransports()); + assertFalse(status.toString().contains("endpoint")); } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/instrumentation/intake/CollectorIntakeAdvertisementService.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/instrumentation/intake/CollectorIntakeAdvertisementService.java index aa43d6f1b6..eca157790d 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/instrumentation/intake/CollectorIntakeAdvertisementService.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/instrumentation/intake/CollectorIntakeAdvertisementService.java @@ -21,12 +21,16 @@ import static org.apache.hertzbeat.common.constants.CommonConstants.COLLECTOR_ST import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus; +import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus.OtlpGatewayState; +import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus.OtlpGatewayTransport; import org.apache.hertzbeat.common.entity.manager.Collector; import org.apache.hertzbeat.common.support.exception.CommonException; import org.apache.hertzbeat.manager.dao.CollectorDao; import org.apache.hertzbeat.manager.pojo.dto.CollectorInstrumentationIntake; import org.apache.hertzbeat.manager.pojo.dto.CollectorInstrumentationIntake.ErrorCode; import org.apache.hertzbeat.manager.pojo.dto.CollectorInstrumentationIntake.Gateway; +import org.apache.hertzbeat.manager.scheduler.runtime.CollectorRuntimeStatusRegistry; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -37,10 +41,15 @@ public class CollectorIntakeAdvertisementService implements CollectorIntakeAdver private final CollectorDao collectorDao; private final CollectorIntakeAdvertisementCodec codec; + private final CollectorRuntimeStatusRegistry runtimeStatuses; - public CollectorIntakeAdvertisementService(CollectorDao collectorDao, CollectorIntakeAdvertisementCodec codec) { + public CollectorIntakeAdvertisementService( + CollectorDao collectorDao, + CollectorIntakeAdvertisementCodec codec, + CollectorRuntimeStatusRegistry runtimeStatuses) { this.collectorDao = collectorDao; this.codec = codec; + this.runtimeStatuses = runtimeStatuses; } @Transactional(rollbackFor = Exception.class) @@ -76,13 +85,35 @@ public class CollectorIntakeAdvertisementService implements CollectorIntakeAdver return CollectorInstrumentationIntake.unavailable( collectorId, ErrorCode.INTAKE_ADVERTISEMENT_INVALID); } - if (request.gateway() == Gateway.COLLECTOR && collector.getStatus() != COLLECTOR_STATUS_ONLINE) { - return CollectorInstrumentationIntake.unavailable( - collectorId, ErrorCode.INTAKE_ADVERTISEMENT_UNAVAILABLE); + if (request.gateway() == Gateway.COLLECTOR) { + if (collector.getStatus() != COLLECTOR_STATUS_ONLINE || !runtimeSupports(collectorId, request)) { + return CollectorInstrumentationIntake.unavailable( + collectorId, ErrorCode.INTAKE_ADVERTISEMENT_UNAVAILABLE); + } } return request.available(collectorId); } + private boolean runtimeSupports(String collectorId, CollectorIntakeAdvertisementRequest request) { + return runtimeStatuses.current(collectorId) + .map(reported -> supports(reported.status(), request)) + .orElse(false); + } + + private boolean supports(ManagedOtelRuntimeStatus status, CollectorIntakeAdvertisementRequest request) { + if (!status.enabled() + || status.state() != ManagedOtelRuntimeStatus.RuntimeState.RUNNING + || status.otlpGateway().state() != OtlpGatewayState.AVAILABLE) { + return false; + } + return request.capabilities().stream().allMatch(capability -> switch (capability) { + case OTLP_HTTP_PROTOBUF -> status.otlpGateway().supportedTransports() + .contains(OtlpGatewayTransport.HTTP_PROTOBUF); + case OTLP_GRPC -> status.otlpGateway().supportedTransports() + .contains(OtlpGatewayTransport.GRPC); + }); + } + private Collector requireCollector(String collectorName) { return collectorDao.findCollectorByName(collectorName) .orElseThrow(() -> new CommonException("Collector not found: " + collectorName)); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/instrumentation/intake/CollectorIntakeAdvertisementServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/instrumentation/intake/CollectorIntakeAdvertisementServiceTest.java index ee0c398a13..717b7565ea 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/instrumentation/intake/CollectorIntakeAdvertisementServiceTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/instrumentation/intake/CollectorIntakeAdvertisementServiceTest.java @@ -30,11 +30,15 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.time.Instant; import java.util.List; import java.util.Optional; +import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus; import org.apache.hertzbeat.common.entity.manager.Collector; import org.apache.hertzbeat.manager.dao.CollectorDao; import org.apache.hertzbeat.manager.pojo.dto.CollectorInstrumentationIntake; +import org.apache.hertzbeat.manager.scheduler.runtime.CollectorRuntimeStatusRegistry; +import org.apache.hertzbeat.manager.scheduler.runtime.CollectorRuntimeStatusRegistry.ReportedStatus; import org.apache.hertzbeat.manager.pojo.dto.CollectorInstrumentationIntake.Capability; import org.apache.hertzbeat.manager.pojo.dto.CollectorInstrumentationIntake.ErrorCode; import org.apache.hertzbeat.manager.pojo.dto.CollectorInstrumentationIntake.Gateway; @@ -57,10 +61,11 @@ class CollectorIntakeAdvertisementServiceTest { .build(); when(collectorDao.findCollectorByName("edge-west")).thenReturn(Optional.of(collector)); CollectorIntakeAdvertisementRequest request = request(Gateway.COLLECTOR); - CollectorIntakeAdvertisementService writer = service(collectorDao); + CollectorRuntimeStatusRegistry runtimeStatuses = runtimeStatuses("edge-west", gatewayAvailable()); + CollectorIntakeAdvertisementService writer = service(collectorDao, runtimeStatuses); CollectorInstrumentationIntake saved = writer.update("edge-west", request); - CollectorInstrumentationIntake reread = service(collectorDao).read(collector); + CollectorInstrumentationIntake reread = service(collectorDao, runtimeStatuses).read(collector); assertEquals(State.AVAILABLE, saved.state()); assertEquals("Authorization", saved.authorizationHeader()); @@ -78,7 +83,7 @@ class CollectorIntakeAdvertisementServiceTest { @Test void rejectsServerOwnedEndpointsAtTheCollectorMutationBoundary() { CollectorDao collectorDao = mock(CollectorDao.class); - CollectorIntakeAdvertisementService service = service(collectorDao); + CollectorIntakeAdvertisementService service = service(collectorDao, mock(CollectorRuntimeStatusRegistry.class)); assertThrows(IllegalArgumentException.class, () -> service.update("edge-west", request(Gateway.SERVER))); verify(collectorDao, never()).save(any()); @@ -87,7 +92,8 @@ class CollectorIntakeAdvertisementServiceTest { @Test void mapsNotAdvertisedInvalidOfflineAndAvailableWithoutExposingStoredText(CapturedOutput output) { CollectorDao collectorDao = mock(CollectorDao.class); - CollectorIntakeAdvertisementService service = service(collectorDao); + CollectorRuntimeStatusRegistry runtimeStatuses = mock(CollectorRuntimeStatusRegistry.class); + CollectorIntakeAdvertisementService service = service(collectorDao, runtimeStatuses); Collector collector = Collector.builder() .name("edge-state") .status(COLLECTOR_STATUS_ONLINE) @@ -105,22 +111,76 @@ class CollectorIntakeAdvertisementServiceTest { assertEquals(ErrorCode.INTAKE_ADVERTISEMENT_UNAVAILABLE, service.read(collector).errorCode()); collector.setStatus(COLLECTOR_STATUS_ONLINE); + assertEquals(ErrorCode.INTAKE_ADVERTISEMENT_UNAVAILABLE, service.read(collector).errorCode()); + + when(runtimeStatuses.current("edge-state")) + .thenReturn(Optional.of(new ReportedStatus(gatewayAvailable(), Instant.now()))); CollectorInstrumentationIntake available = service.read(collector); assertEquals(State.AVAILABLE, available.state()); assertEquals(Gateway.COLLECTOR, available.gateway()); assertEquals("Authorization", available.authorizationHeader()); } - private CollectorIntakeAdvertisementService service(CollectorDao collectorDao) { - return new CollectorIntakeAdvertisementService(collectorDao, new CollectorIntakeAdvertisementCodec()); + @Test + void rejectsAnAdvertisedProtocolThatTheCurrentRuntimeDoesNotReport() { + Collector collector = Collector.builder() + .name("edge-protocol") + .status(COLLECTOR_STATUS_ONLINE) + .build(); + collector.setInstrumentationIntake(new CollectorIntakeAdvertisementCodec().encode( + request(Gateway.COLLECTOR, Capability.OTLP_HTTP_PROTOBUF))); + CollectorIntakeAdvertisementService service = service( + mock(CollectorDao.class), runtimeStatuses("edge-protocol", gatewayAvailable())); + + CollectorInstrumentationIntake intake = service.read(collector); + + assertEquals(State.UNAVAILABLE, intake.state()); + assertEquals(ErrorCode.INTAKE_ADVERTISEMENT_UNAVAILABLE, intake.errorCode()); + } + + private CollectorIntakeAdvertisementService service( + CollectorDao collectorDao, CollectorRuntimeStatusRegistry runtimeStatuses) { + return new CollectorIntakeAdvertisementService( + collectorDao, new CollectorIntakeAdvertisementCodec(), runtimeStatuses); + } + + private CollectorRuntimeStatusRegistry runtimeStatuses( + String collectorId, ManagedOtelRuntimeStatus runtimeStatus) { + CollectorRuntimeStatusRegistry registry = mock(CollectorRuntimeStatusRegistry.class); + when(registry.current(collectorId)) + .thenReturn(Optional.of(new ReportedStatus(runtimeStatus, Instant.now()))); + return registry; + } + + private ManagedOtelRuntimeStatus gatewayAvailable() { + return new ManagedOtelRuntimeStatus( + ManagedOtelRuntimeStatus.CURRENT_SCHEMA_VERSION, + true, + ManagedOtelRuntimeStatus.RuntimeState.RUNNING, + 1, + 1, + -1, + ManagedOtelRuntimeStatus.IntakeCredentialState.CONFIGURED, + 0, + Instant.now(), + "", + ManagedOtelRuntimeStatus.FailureCode.NONE, + ManagedOtelRuntimeStatus.RuntimeTelemetry.unavailable(false), + List.of(), + ManagedOtelRuntimeStatus.OtlpGatewayStatus.available( + List.of(ManagedOtelRuntimeStatus.OtlpGatewayTransport.GRPC))); } private CollectorIntakeAdvertisementRequest request(Gateway gateway) { + return request(gateway, Capability.OTLP_GRPC); + } + + private CollectorIntakeAdvertisementRequest request(Gateway gateway, Capability capability) { return new CollectorIntakeAdvertisementRequest( 1, gateway, - List.of(Capability.OTLP_GRPC), - null, - "https://collector.example.test:4317"); + List.of(capability), + capability == Capability.OTLP_HTTP_PROTOBUF ? "https://collector.example.test:4318" : null, + capability == Capability.OTLP_GRPC ? "https://collector.example.test:4317" : null); } } From 7ce055d2ce815a2c0f46a389cfdd4445d5c819bb Mon Sep 17 00:00:00 2001 From: Logic Date: Thu, 6 Aug 2026 14:42:43 +0800 Subject: [PATCH 04/71] Treat embedded collector as server-local --- .../ui/runtime/UiRuntimeStatusQueryService.java | 9 ++++++++- .../ui/runtime/UiRuntimeStatusQueryServiceTest.java | 12 ++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/ui/runtime/UiRuntimeStatusQueryService.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/ui/runtime/UiRuntimeStatusQueryService.java index dcbb34d807..a6113bf071 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/ui/runtime/UiRuntimeStatusQueryService.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/ui/runtime/UiRuntimeStatusQueryService.java @@ -18,6 +18,7 @@ package org.apache.hertzbeat.manager.ui.runtime; import static org.apache.hertzbeat.common.constants.CommonConstants.COLLECTOR_STATUS_ONLINE; +import static org.apache.hertzbeat.common.constants.CommonConstants.MAIN_COLLECTOR_NODE; import java.time.Clock; import java.time.Instant; @@ -96,10 +97,16 @@ public class UiRuntimeStatusQueryService implements UiRuntimeStatusQuery { boolean enabledRuntimeFailure = false; Instant lastReportedAt = null; for (CollectorStatusInventory collector : inventory) { - boolean collectorOnline = collector.getStatus() == COLLECTOR_STATUS_ONLINE; + boolean embedded = MAIN_COLLECTOR_NODE.equals(collector.getName()); + // The embedded Java Collector shares the Server process lifetime and has no external heartbeat. + boolean collectorOnline = embedded || collector.getStatus() == COLLECTOR_STATUS_ONLINE; if (collectorOnline) { online++; } + if (embedded) { + runtimeHealthy++; + continue; + } ReportedStatus report = runtimeStatusRegistry.current(collector.getName()).orElse(null); if (report == null) { continue; diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/ui/runtime/UiRuntimeStatusQueryServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/ui/runtime/UiRuntimeStatusQueryServiceTest.java index e20219235e..96cb360244 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/ui/runtime/UiRuntimeStatusQueryServiceTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/ui/runtime/UiRuntimeStatusQueryServiceTest.java @@ -19,6 +19,7 @@ package org.apache.hertzbeat.manager.ui.runtime; import static org.apache.hertzbeat.common.constants.CommonConstants.COLLECTOR_STATUS_OFFLINE; import static org.apache.hertzbeat.common.constants.CommonConstants.COLLECTOR_STATUS_ONLINE; +import static org.apache.hertzbeat.common.constants.CommonConstants.MAIN_COLLECTOR_NODE; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; @@ -95,6 +96,17 @@ class UiRuntimeStatusQueryServiceTest { assertCollectors(service().current(), State.AVAILABLE, 2, 2, 0); } + @Test + void treatsTheServerEmbeddedCollectorAsOnlineWithTheServerProcess() { + when(collectorDao.findStatusInventory()).thenReturn( + List.of(inventory(MAIN_COLLECTOR_NODE, COLLECTOR_STATUS_OFFLINE))); + + RuntimeStatusResponse response = service().current(); + + assertCollectors(response, State.AVAILABLE, 1, 1, 1); + assertNull(response.collectors().lastReportedAt()); + } + @Test void countsOnlyFreshEnabledRunningFailureFreeManagedRuntimes() { when(collectorDao.findStatusInventory()).thenReturn(List.of( From 8b0b690651f7c474d8f886b8147d909c233e9e4b Mon Sep 17 00:00:00 2001 From: Logic Date: Thu, 6 Aug 2026 15:10:01 +0800 Subject: [PATCH 05/71] Allow built-in monitor definition overrides --- .../MonitorDefinitionCommandService.java | 9 +++-- .../definition/MonitorDefinitionService.java | 15 ++++--- .../MonitorDefinitionCommandPortTest.java | 40 +++++++++++++++++-- .../MonitorDefinitionServiceTest.java | 10 ++--- 4 files changed, 55 insertions(+), 19 deletions(-) diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/monitor/definition/MonitorDefinitionCommandService.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/monitor/definition/MonitorDefinitionCommandService.java index ee4eb2f4d6..8c01771a24 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/monitor/definition/MonitorDefinitionCommandService.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/monitor/definition/MonitorDefinitionCommandService.java @@ -48,9 +48,6 @@ public class MonitorDefinitionCommandService implements MonitorDefinitionCommand return executor.executeSerialized(state -> { MonitorDefinitionSource previous = require(app, state.readAll()); Job parsed = parse(state, definition); - if (MonitorDefinitionRevision.origin(previous) == MonitorDefinitionOrigin.BUILTIN) { - throw failure(MonitorDefinitionErrorCode.IMMUTABLE); - } if (!previous.job().getApp().equals(app) || !previous.job().getApp().equals(parsed.getApp())) { throw failure(MonitorDefinitionErrorCode.UPDATE_TARGET_MISMATCH); } @@ -134,6 +131,12 @@ public class MonitorDefinitionCommandService implements MonitorDefinitionCommand if (previous == null) { state.remove(parsed.getApp()); state.publishRemoval(parsed.getApp()); + } else if (MonitorDefinitionRevision.origin(previous) == MonitorDefinitionOrigin.BUILTIN) { + // A built-in edit creates a persisted override. Rollback must remove that override + // so the packaged definition becomes effective again instead of persisting a copy. + state.remove(previous.job().getApp()); + state.publishRemoval(previous.job().getApp()); + state.updateRuntime(previous.job()); } else { state.save(previous.job().getApp(), previous.definition()); state.publish(previous.job(), previous.definition()); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/monitor/definition/MonitorDefinitionService.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/monitor/definition/MonitorDefinitionService.java index 06f6ad6fce..a64321f160 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/monitor/definition/MonitorDefinitionService.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/monitor/definition/MonitorDefinitionService.java @@ -105,10 +105,9 @@ public class MonitorDefinitionService { throw new MonitorDefinitionException(MonitorDefinitionErrorCode.UPDATE_TARGET_MISMATCH); } MonitorDefinitionOrigin origin = MonitorDefinitionRevision.origin(target); - if (origin == MonitorDefinitionOrigin.BUILTIN) { - throw new MonitorDefinitionException(MonitorDefinitionErrorCode.IMMUTABLE); - } - return validationResponse(canonicalApp, origin); + return validationResponse( + canonicalApp, + origin == MonitorDefinitionOrigin.BUILTIN ? MonitorDefinitionOrigin.OVERRIDE : origin); } private Job parseAndValidate(String definition) { @@ -139,11 +138,15 @@ public class MonitorDefinitionService { private static MonitorDefinitionCatalogItem catalogItem(MonitorDefinitionSource source, String lang) { MonitorDefinitionOrigin origin = MonitorDefinitionRevision.origin(source); - boolean mutable = origin != MonitorDefinitionOrigin.BUILTIN; String app = source.job().getApp(); String label = CommonUtil.getLangMappingValueFromI18nMap(normalizeLang(lang), source.job().getName()); return new MonitorDefinitionCatalogItem( - app, label == null ? app : label, origin, mutable, mutable, MonitorDefinitionRevision.from(source)); + app, + label == null ? app : label, + origin, + true, + origin != MonitorDefinitionOrigin.BUILTIN, + MonitorDefinitionRevision.from(source)); } private static MonitorDefinitionDetailResponse detail(MonitorDefinitionSource source, String lang) { diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/monitor/definition/MonitorDefinitionCommandPortTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/monitor/definition/MonitorDefinitionCommandPortTest.java index 9ff9396f6c..81da2acbfa 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/monitor/definition/MonitorDefinitionCommandPortTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/monitor/definition/MonitorDefinitionCommandPortTest.java @@ -117,7 +117,7 @@ class MonitorDefinitionCommandPortTest { } @Test - void updateRequiresMutableExactIdentityAndCurrentRevision() { + void updateRequiresExactIdentityAndCurrentRevision() { MonitorDefinitionSource created = commandService.create(customDefinition("write-update", "one")); String revision = MonitorDefinitionRevision.from(created); String updatedDefinition = customDefinition("write-update", "two"); @@ -131,10 +131,24 @@ class MonitorDefinitionCommandPortTest { assertError(MonitorDefinitionErrorCode.UPDATE_TARGET_MISMATCH, () -> commandService.update("write-update", MonitorDefinitionRevision.from(updated), customDefinition("different-app", "three"))); + } + + @Test + void updatingBuiltinCreatesDeletableOverrideWithoutMutatingPackagedDefinition() { MonitorDefinitionSource builtin = source("jvm"); - assertError(MonitorDefinitionErrorCode.IMMUTABLE, - () -> commandService.update("jvm", MonitorDefinitionRevision.from(builtin), - customDefinition("jvm", "x"))); + String builtinDefinition = builtin.definition(); + String overrideDefinition = customDefinition("jvm", "override-from-editor"); + + MonitorDefinitionSource override = commandService.update( + "jvm", MonitorDefinitionRevision.from(builtin), overrideDefinition); + + assertEquals(MonitorDefinitionOrigin.OVERRIDE, MonitorDefinitionRevision.origin(override)); + assertEquals(overrideDefinition, override.definition()); + assertEquals(MonitorDefinitionDeleteDisposition.BUILTIN_RESTORED, + commandService.delete("jvm", MonitorDefinitionRevision.from(override)).disposition()); + MonitorDefinitionSource restored = source("jvm"); + assertEquals(MonitorDefinitionOrigin.BUILTIN, MonitorDefinitionRevision.origin(restored)); + assertEquals(builtinDefinition, restored.definition()); } @Test @@ -241,6 +255,24 @@ class MonitorDefinitionCommandPortTest { assertEquals(previous.definition(), source("write-update-rollback").definition()); } + @Test + void builtinOverrideRuntimeFailureRestoresPackagedDefinitionWithoutPersistedOverride() { + MonitorDefinitionSource builtin = source("jvm"); + doThrow(new IllegalStateException("new runtime failed")) + .doNothing() + .when(monitorService).updateAppCollectJob(any(Job.class)); + + assertError(MonitorDefinitionErrorCode.RUNTIME_UPDATE_FAILED, + () -> commandService.update("jvm", MonitorDefinitionRevision.from(builtin), + customDefinition("jvm", "override-runtime-failure"))); + + verify(defineDao).deleteById("jvm"); + verify(monitorService, times(2)).updateAppCollectJob(any(Job.class)); + MonitorDefinitionSource restored = source("jvm"); + assertEquals(MonitorDefinitionOrigin.BUILTIN, MonitorDefinitionRevision.origin(restored)); + assertEquals(builtin.definition(), restored.definition()); + } + @Test void updateRuntimeCompensationFailureReportsStateUncertain() { MonitorDefinitionSource previous = commandService.create(customDefinition("write-runtime-uncertain", "one")); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/monitor/definition/MonitorDefinitionServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/monitor/definition/MonitorDefinitionServiceTest.java index e16f89afa7..503ef3432a 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/monitor/definition/MonitorDefinitionServiceTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/monitor/definition/MonitorDefinitionServiceTest.java @@ -59,7 +59,7 @@ class MonitorDefinitionServiceTest { response.items().stream().map(MonitorDefinitionCatalogItem::origin).toList()); assertTrue(response.items().get(0).editable()); assertTrue(response.items().get(1).deletable()); - assertFalse(response.items().get(2).editable()); + assertTrue(response.items().get(2).editable()); assertFalse(response.items().get(2).deletable()); } @@ -139,7 +139,7 @@ class MonitorDefinitionServiceTest { } @Test - void updateValidationRejectsMismatchedMissingAndBuiltinTargets() { + void updateValidationRejectsMismatchedAndMissingTargetsButAllowsBuiltinOverride() { when(sourceReader.readAll()).thenReturn(List.of( source("custom-app", "Custom", false, true), source("jvm", "JVM", true, false))); @@ -155,10 +155,8 @@ class MonitorDefinitionServiceTest { assertThrows(MonitorDefinitionException.class, () -> service.validate(request(MonitorDefinitionOperation.UPDATE, "missing", "app: missing"))) .errorCode()); - assertEquals(MonitorDefinitionErrorCode.IMMUTABLE, - assertThrows(MonitorDefinitionException.class, - () -> service.validate(request(MonitorDefinitionOperation.UPDATE, "jvm", "app: jvm"))) - .errorCode()); + assertEquals(MonitorDefinitionOrigin.OVERRIDE, + service.validate(request(MonitorDefinitionOperation.UPDATE, "jvm", "app: jvm")).origin()); assertEquals(MonitorDefinitionErrorCode.EXPECTED_APP_REQUIRED, assertThrows(MonitorDefinitionException.class, () -> service.validate(request(MonitorDefinitionOperation.UPDATE, " ", "app: jvm"))) From 7cc18f64f37a26ac13716b0791196616d0a7565e Mon Sep 17 00:00:00 2001 From: Logic Date: Fri, 7 Aug 2026 18:02:17 +0800 Subject: [PATCH 06/71] Omit null entries from alert payload maps --- .../common/entity/alerter/GroupAlert.java | 4 ++ .../common/entity/alerter/SingleAlert.java | 3 + .../alerter/AlertEntitySerializationTest.java | 60 +++++++++++++++++++ 3 files changed, 67 insertions(+) create mode 100644 hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/entity/alerter/AlertEntitySerializationTest.java diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/alerter/GroupAlert.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/alerter/GroupAlert.java index 84fb7b3251..ae9d7ae85e 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/alerter/GroupAlert.java +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/alerter/GroupAlert.java @@ -19,6 +19,7 @@ package org.apache.hertzbeat.common.entity.alerter; import static io.swagger.v3.oas.annotations.media.Schema.AccessMode.READ_ONLY; import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonInclude; import io.swagger.v3.oas.annotations.media.Schema; import jakarta.persistence.Column; import jakarta.persistence.Convert; @@ -72,16 +73,19 @@ public class GroupAlert { @Schema(title = "Group Labels", example = "{\"alertname\": \"HighCPUUsage\"}") @Convert(converter = JsonMapAttributeConverter.class) @Column(length = 2048) + @JsonInclude(content = JsonInclude.Include.NON_NULL) private Map groupLabels; @Schema(title = "Common Labels", example = "{\"alertname\": \"HighCPUUsage\", \"instance\": \"server1\", \"severity\": \"critical\"}") @Convert(converter = JsonMapAttributeConverter.class) @Column(length = 2048) + @JsonInclude(content = JsonInclude.Include.NON_NULL) private Map commonLabels; @Schema(title = "Common Annotations", example = "{\"summary\": \"High CPU usage detected\", \"description\": \"CPU usage is back to normal for server1\"}") @Convert(converter = JsonMapAttributeConverter.class) @Column(columnDefinition = "TEXT") + @JsonInclude(content = JsonInclude.Include.NON_NULL) private Map commonAnnotations; @Schema(title = "Alert Fingerprints", example = "[\"dxsdfdsf\"]") diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/alerter/SingleAlert.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/alerter/SingleAlert.java index 617655c97f..d15d618337 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/alerter/SingleAlert.java +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/alerter/SingleAlert.java @@ -19,6 +19,7 @@ package org.apache.hertzbeat.common.entity.alerter; import static io.swagger.v3.oas.annotations.media.Schema.AccessMode.READ_ONLY; import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonInclude; import io.swagger.v3.oas.annotations.media.Schema; import jakarta.persistence.Column; import jakarta.persistence.Convert; @@ -67,11 +68,13 @@ public class SingleAlert { @Schema(title = "Labels", example = "{\"alertname\": \"HighCPUUsage\", \"priority\": \"critical\", \"instance\": \"343483943\"}") @Convert(converter = JsonMapAttributeConverter.class) @Column(length = 2048) + @JsonInclude(content = JsonInclude.Include.NON_NULL) private Map labels; @Schema(title = "Annotations", example = "{\"summary\": \"High CPU usage detected\"}") @Convert(converter = JsonMapAttributeConverter.class) @Column(length = 4096) + @JsonInclude(content = JsonInclude.Include.NON_NULL) private Map annotations; @Schema(title = "Content", example = "CPU usage is above 80% for the last 5 minutes on instance server1.example.com.") diff --git a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/entity/alerter/AlertEntitySerializationTest.java b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/entity/alerter/AlertEntitySerializationTest.java new file mode 100644 index 0000000000..b32a9e2012 --- /dev/null +++ b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/entity/alerter/AlertEntitySerializationTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.entity.alerter; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.hertzbeat.common.util.JsonUtil; +import org.junit.jupiter.api.Test; +import tools.jackson.databind.JsonNode; + +class AlertEntitySerializationTest { + + @Test + void shouldOmitNullMapEntriesFromAlertApiPayloads() { + Map labels = new HashMap<>(); + labels.put("alertname", "CollectorUnavailable"); + labels.put("collectorVersion", null); + SingleAlert singleAlert = SingleAlert.builder() + .labels(labels) + .annotations(new HashMap<>(labels)) + .build(); + GroupAlert groupAlert = GroupAlert.builder() + .groupLabels(new HashMap<>(labels)) + .commonLabels(new HashMap<>(labels)) + .commonAnnotations(new HashMap<>(labels)) + .alerts(List.of(singleAlert)) + .build(); + + JsonNode payload = JsonUtil.fromJson(JsonUtil.toJson(groupAlert)); + + assertStringMapWithoutNullEntry(payload.path("groupLabels")); + assertStringMapWithoutNullEntry(payload.path("commonLabels")); + assertStringMapWithoutNullEntry(payload.path("commonAnnotations")); + assertStringMapWithoutNullEntry(payload.path("alerts").path(0).path("labels")); + assertStringMapWithoutNullEntry(payload.path("alerts").path(0).path("annotations")); + } + + private void assertStringMapWithoutNullEntry(JsonNode map) { + assertEquals("CollectorUnavailable", map.path("alertname").textValue()); + assertFalse(map.has("collectorVersion")); + } +} From 3c79e5c31ea761c5da51df1e0bacad30c764c8c1 Mon Sep 17 00:00:00 2001 From: Logic Date: Fri, 7 Aug 2026 22:58:49 +0800 Subject: [PATCH 07/71] Define setup and deployment contracts --- .../setup/api/DeploymentApiContract.java | 154 +++++ .../manager/setup/api/DeploymentWorkflow.java | 39 ++ .../manager/setup/api/SetupApiContract.java | 595 ++++++++++++++++++ .../manager/setup/api/SetupHttpContract.java | 33 + .../manager/setup/api/SetupWorkflow.java | 57 ++ .../setup/api/DeploymentApiContractTest.java | 106 ++++ .../setup/api/SetupApiContractTest.java | 215 +++++++ .../setup/api/SetupHttpContractTest.java | 68 ++ 8 files changed, 1267 insertions(+) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContract.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentWorkflow.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupHttpContract.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupWorkflow.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContractTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupHttpContractTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContract.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContract.java new file mode 100644 index 0000000000..89a451dcd3 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContract.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import com.fasterxml.jackson.annotation.JsonValue; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.PositiveOrZero; +import java.time.Instant; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ManagementDatabaseSummary; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreSummary; + +/** Authenticated deployment configuration and H2 migration contract. */ +public final class DeploymentApiContract { + + public static final String DEPLOYMENT_PATH = "/api/config/deployment"; + public static final String VALIDATE_PATH = "/api/config/deployment/validate"; + public static final String MIGRATION_PATH = "/api/config/deployment/metadata-migrations"; + public static final String MIGRATION_OPERATION_PATH = + "/api/config/deployment/metadata-migrations/{operationId}"; + public static final String ACTIVATE_PATH = + "/api/config/deployment/metadata-migrations/{operationId}/activate"; + + private DeploymentApiContract() { + } + + private interface WireValue { + + @JsonValue + String value(); + } + + /** Supported external metadata migration target. */ + public enum MigrationTarget implements WireValue { + MYSQL("mysql", MetadataDatabaseKind.MYSQL), + POSTGRESQL("postgresql", MetadataDatabaseKind.POSTGRESQL); + + private final String value; + private final MetadataDatabaseKind databaseKind; + + MigrationTarget(String value, MetadataDatabaseKind databaseKind) { + this.value = value; + this.databaseKind = databaseKind; + } + + @Override + public String value() { + return value; + } + + MetadataDatabaseKind databaseKind() { + return databaseKind; + } + } + + /** Migration verification lifecycle. */ + public enum VerificationState implements WireValue { + PENDING("pending"), + RUNNING("running"), + SUCCEEDED("succeeded"), + FAILED("failed"); + + private final String value; + + VerificationState(String value) { + this.value = value; + } + + @Override + public String value() { + return value; + } + } + + /** Secret-free authenticated deployment view. */ + public record DeploymentView( + @NotNull Instant observedAt, + @NotNull @Valid ManagementDatabaseSummary managementDatabase, + @NotNull @Valid TelemetryStoreSummary telemetryStore, + @NotNull ApplyMode applyMode, + boolean maintenanceMode, + boolean migrationAllowed) { + } + + /** H2-to-external-database migration input. */ + public record MetadataMigrationRequest( + @NotNull MigrationTarget target, + @NotNull @Valid MetadataDatabaseConfiguration targetDatabase, + @NotNull ApplyMode applyMode) { + + public MetadataMigrationRequest { + if (target == null || targetDatabase == null || target.databaseKind() != targetDatabase.kind()) { + throw new IllegalArgumentException("Migration target and target database kind must match"); + } + } + } + + /** Safe migration operation view; table identities and verification details are intentionally absent. */ + public record MigrationView( + @NotBlank String operationId, + @NotNull SetupOperationState state, + @NotNull MetadataDatabaseKind source, + @NotNull MigrationTarget target, + @NotNull SetupPhase phase, + @NotNull Instant createdAt, + Instant startedAt, + Instant completedAt, + @PositiveOrZero long tablesTotal, + @PositiveOrZero long tablesCopied, + @NotNull VerificationState verificationState, + SetupErrorCode errorCode, + boolean activationAvailable, + boolean externalApplyRequired) { + + public MigrationView { + if (source != MetadataDatabaseKind.H2) { + throw new IllegalArgumentException("Migration source must be H2"); + } + if (phase != SetupPhase.MIGRATION_IN_PROGRESS) { + throw new IllegalArgumentException("Migration view must report migration in progress"); + } + if (tablesTotal < 0 || tablesCopied < 0 || tablesCopied > tablesTotal) { + throw new IllegalArgumentException("Migration table counts are inconsistent"); + } + } + } + + /** Explicit migration activation input. */ + public record ActivateMigrationRequest( + @NotNull SetupOperationState expectedState) { + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentWorkflow.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentWorkflow.java new file mode 100644 index 0000000000..bce6c4b459 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentWorkflow.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.ActivateMigrationRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentView; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MetadataMigrationRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationView; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidateRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationResponse; + +/** Authenticated deployment boundary implemented by a later migration engine milestone. */ +public interface DeploymentWorkflow { + + DeploymentView deployment(); + + ValidationResponse validate(ValidateRequest request); + + MigrationView migrate(MetadataMigrationRequest request); + + MigrationView migration(String operationId); + + MigrationView activate(String operationId, ActivateMigrationRequest request); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java new file mode 100644 index 0000000000..8b5818ffd7 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java @@ -0,0 +1,595 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonValue; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Positive; +import jakarta.validation.constraints.PositiveOrZero; +import java.time.Instant; +import java.util.List; +import java.util.Objects; + +/** + * Unversioned wire and domain contract for first-install setup. + * + *

The contract deliberately contains no persistence, database creation, or application restart behavior. + */ +public final class SetupApiContract { + + public static final String STATUS_PATH = "/api/setup/status"; + public static final String UNLOCK_PATH = "/api/setup/unlock"; + public static final String VALIDATE_PATH = "/api/setup/validate"; + public static final String CONFIGURATION_PATH = "/api/setup/configuration"; + public static final String OPERATION_PATH = "/api/setup/operations/{operationId}"; + public static final String ADMINISTRATOR_PATH = "/api/setup/administrator"; + public static final String OPTIONS_PATH = "/api/setup/options"; + public static final String EXPORT_PATH = "/api/setup/export"; + public static final String COMPLETE_PATH = "/api/setup/complete"; + + private SetupApiContract() { + } + + private interface WireValue { + + @JsonValue + String value(); + } + + /** Setup workflow phase. */ + public enum SetupPhase implements WireValue { + CONFIGURATION_REQUIRED("configuration_required"), + EXTERNAL_APPLY_REQUIRED("external_apply_required"), + APPLICATION_STARTING("application_starting"), + ADMINISTRATOR_REQUIRED("administrator_required"), + OPTIONAL_CONFIGURATION("optional_configuration"), + COMPLETE("complete"), + RECOVERY_REQUIRED("recovery_required"), + MIGRATION_IN_PROGRESS("migration_in_progress"); + + private final String value; + + SetupPhase(String value) { + this.value = value; + } + + @Override + public String value() { + return value; + } + } + + /** Effective configuration origin. */ + public enum ConfigSource implements WireValue { + BUILT_IN_DEFAULT("built_in_default"), + UI_MANAGED("ui_managed"), + EXTERNAL_FILE("external_file"), + ENVIRONMENT("environment"), + SYSTEM_PROPERTY("system_property"), + COMMAND_LINE("command_line"); + + private final String value; + + ConfigSource(String value) { + this.value = value; + } + + @Override + public String value() { + return value; + } + } + + /** How validated configuration becomes active. */ + public enum ApplyMode implements WireValue { + MANAGED_WRITE("managed_write"), + EXTERNAL_APPLY("external_apply"); + + private final String value; + + ApplyMode(String value) { + this.value = value; + } + + @Override + public String value() { + return value; + } + } + + /** Setup access boundary. */ + public enum SetupAccess implements WireValue { + LOCAL("local"), + LOCKED("locked"), + UNLOCKED("unlocked"); + + private final String value; + + SetupAccess(String value) { + this.value = value; + } + + @Override + public String value() { + return value; + } + } + + /** Asynchronous operation lifecycle. */ + public enum SetupOperationState implements WireValue { + PENDING("pending"), + RUNNING("running"), + AWAITING_EXTERNAL_APPLY("awaiting_external_apply"), + AWAITING_RESTART("awaiting_restart"), + SUCCEEDED("succeeded"), + FAILED("failed"), + ROLLED_BACK("rolled_back"); + + private final String value; + + SetupOperationState(String value) { + this.value = value; + } + + @Override + public String value() { + return value; + } + } + + /** Supported metadata database. */ + public enum MetadataDatabaseKind implements WireValue { + H2("h2"), + MYSQL("mysql"), + POSTGRESQL("postgresql"); + + private final String value; + + MetadataDatabaseKind(String value) { + this.value = value; + } + + @Override + public String value() { + return value; + } + } + + /** Supported telemetry store. */ + public enum TelemetryStoreKind implements WireValue { + GREPTIME("greptime"); + + private final String value; + + TelemetryStoreKind(String value) { + this.value = value; + } + + @Override + public String value() { + return value; + } + } + + /** Independently validated configuration section. */ + public enum ValidationSection implements WireValue { + METADATA_DATABASE("metadata_database"), + TELEMETRY_STORE("telemetry_store"), + PUBLIC_ACCESS("public_access"), + MAIL("mail"); + + private final String value; + + ValidationSection(String value) { + this.value = value; + } + + @Override + public String value() { + return value; + } + } + + /** Mail transport security. */ + public enum MailSecurity implements WireValue { + NONE("none"), + STARTTLS("starttls"), + TLS("tls"); + + private final String value; + + MailSecurity(String value) { + this.value = value; + } + + @Override + public String value() { + return value; + } + } + + /** Safe, stable error classification. */ + public enum SetupErrorCode implements WireValue { + SETUP_COMPLETE("setup_complete"), + SETUP_LOCKED("setup_locked"), + SETUP_CODE_INVALID("setup_code_invalid"), + SETUP_CODE_EXPIRED("setup_code_expired"), + SETUP_RATE_LIMITED("setup_rate_limited"), + CONFIG_READ_ONLY("config_read_only"), + CONFIG_WRITE_FAILED("config_write_failed"), + CONFIG_RECOVERY_REQUIRED("config_recovery_required"), + METADATA_CONNECTION_FAILED("metadata_connection_failed"), + METADATA_KIND_UNSUPPORTED("metadata_kind_unsupported"), + METADATA_SCHEMA_MISMATCH("metadata_schema_mismatch"), + METADATA_INSUFFICIENT_PRIVILEGES("metadata_insufficient_privileges"), + TELEMETRY_CONNECTION_FAILED("telemetry_connection_failed"), + PUBLIC_ADDRESS_INVALID("public_address_invalid"), + MAIL_CONNECTION_FAILED("mail_connection_failed"), + ADMINISTRATOR_ALREADY_CONFIGURED("administrator_already_configured"), + ADMINISTRATOR_USERNAME_INVALID("administrator_username_invalid"), + OPERATION_NOT_FOUND("operation_not_found"), + OPERATION_CONFLICT("operation_conflict"), + MIGRATION_SOURCE_UNSUPPORTED("migration_source_unsupported"), + MIGRATION_TARGET_NOT_EMPTY("migration_target_not_empty"), + MIGRATION_MULTI_NODE_UNSUPPORTED("migration_multi_node_unsupported"), + MIGRATION_COPY_FAILED("migration_copy_failed"), + MIGRATION_VERIFICATION_FAILED("migration_verification_failed"), + MIGRATION_ACTIVATION_FAILED("migration_activation_failed"), + RESTART_FAILED("restart_failed"); + + private final String value; + + SetupErrorCode(String value) { + this.value = value; + } + + @Override + public String value() { + return value; + } + } + + /** Stable warning codes acknowledged during setup completion. */ + public enum SetupWarningCode implements WireValue { + EXTERNAL_APPLY_REQUIRED("external_apply_required"), + RESTART_REQUIRED("restart_required"), + PUBLIC_ADDRESS_PLAINTEXT("public_address_plaintext"), + MAIL_SECURITY_NONE("mail_security_none"); + + private final String value; + + SetupWarningCode(String value) { + this.value = value; + } + + @Override + public String value() { + return value; + } + } + + /** Safe setup status. */ + public record StatusResponse( + @NotNull SetupPhase phase, + @NotNull Instant observedAt, + @NotNull SetupAccess access, + @NotNull ApplyMode applyMode, + boolean writableManagedConfig, + String operationId, + SetupErrorCode errorCode, + @NotNull @Valid ManagementDatabaseSummary managementDatabase, + @NotNull @Valid TelemetryStoreSummary telemetryStore, + boolean administratorConfigured, + @NotNull @Valid OptionalConfigurationSummary optional) { + } + + /** Secret-free metadata database summary. */ + public record ManagementDatabaseSummary( + MetadataDatabaseKind kind, + boolean configured, + @NotNull ConfigSource source, + boolean restartRequired) { + } + + /** Secret-free telemetry store summary. */ + public record TelemetryStoreSummary( + @NotNull TelemetryStoreKind kind, + boolean configured, + @NotNull ConfigSource source, + boolean restartRequired) { + + public TelemetryStoreSummary { + if (kind != TelemetryStoreKind.GREPTIME) { + throw new IllegalArgumentException("Only Greptime telemetry storage is supported"); + } + } + } + + /** Secret-free optional configuration status. */ + public record OptionalConfigurationSummary( + boolean publicAccessConfigured, + boolean serverOtlpHttpConfigured, + boolean serverOtlpGrpcConfigured, + boolean retentionConfigured, + boolean mailConfigured) { + } + + /** One-time unlock proof. */ + public record UnlockRequest( + @NotBlank @JsonProperty(access = JsonProperty.Access.WRITE_ONLY) String code) { + + @Override + public String toString() { + return "UnlockRequest[code=]"; + } + } + + /** Successful unlock result. */ + public record UnlockResponse(@NotNull SetupAccess access, @NotNull Instant expiresAt) { + + public UnlockResponse { + if (access != SetupAccess.UNLOCKED) { + throw new IllegalArgumentException("An unlock response must report unlocked access"); + } + } + } + + /** Metadata database input. */ + public record MetadataDatabaseConfiguration( + @NotNull MetadataDatabaseKind kind, + @NotBlank String jdbcUrl, + @NotBlank String username, + @NotBlank @JsonProperty(access = JsonProperty.Access.WRITE_ONLY) String password) { + + @Override + public String toString() { + return "MetadataDatabaseConfiguration[kind=" + kind + ", password=]"; + } + } + + /** Telemetry store input. */ + public record TelemetryStoreConfiguration( + @NotNull TelemetryStoreKind kind, + @NotBlank String grpcEndpoints, + @NotBlank String httpEndpoint, + @NotBlank String database, + @NotBlank String username, + @NotBlank @JsonProperty(access = JsonProperty.Access.WRITE_ONLY) String password) { + + public TelemetryStoreConfiguration { + if (kind != TelemetryStoreKind.GREPTIME) { + throw new IllegalArgumentException("Only Greptime telemetry storage is supported"); + } + } + + @Override + public String toString() { + return "TelemetryStoreConfiguration[kind=" + kind + ", password=]"; + } + } + + /** Public endpoint input; HTTP and HTTPS are both contractually valid. */ + public record PublicAccessConfiguration( + String publicBaseUrl, + String serverOtlpHttpEndpoint, + String serverOtlpGrpcEndpoint) { + } + + /** Mail input. */ + public record MailConfiguration( + @NotBlank String host, + @Positive int port, + @NotNull MailSecurity security, + String username, + @JsonProperty(access = JsonProperty.Access.WRITE_ONLY) String password, + @NotBlank String fromAddress) { + + public MailConfiguration { + if (port <= 0) { + throw new IllegalArgumentException("Mail port must be positive"); + } + } + + @Override + public String toString() { + return "MailConfiguration[host=" + host + ", port=" + port + ", security=" + security + + ", password=]"; + } + } + + /** Exactly one configuration section validation input. */ + public record ValidateRequest( + @NotNull ValidationSection section, + @Valid MetadataDatabaseConfiguration managementDatabase, + @Valid TelemetryStoreConfiguration telemetryStore, + @Valid PublicAccessConfiguration publicAccess, + @Valid MailConfiguration mail) { + + public ValidateRequest { + Objects.requireNonNull(section, "section"); + int supplied = countPresent(managementDatabase, telemetryStore, publicAccess, mail); + boolean matches = switch (section) { + case METADATA_DATABASE -> managementDatabase != null; + case TELEMETRY_STORE -> telemetryStore != null; + case PUBLIC_ACCESS -> publicAccess != null; + case MAIL -> mail != null; + }; + if (supplied != 1 || !matches) { + throw new IllegalArgumentException("Exactly the selected validation section must be supplied"); + } + } + } + + /** Safe validation result. */ + public record ValidationResponse( + boolean valid, + @NotNull Instant observedAt, + SetupErrorCode errorCode, + @NotNull List warnings) { + + public ValidationResponse { + warnings = List.copyOf(warnings); + } + } + + /** Validated required configuration input. */ + public record ConfigurationRequest( + @NotNull SetupPhase expectedPhase, + @NotNull ApplyMode applyMode, + @NotNull @Valid MetadataDatabaseConfiguration managementDatabase, + @NotNull @Valid TelemetryStoreConfiguration telemetryStore) { + } + + /** Configuration operation acknowledgement. */ + public record ConfigurationResponse( + @NotBlank String operationId, + @NotNull SetupOperationState state, + @NotNull SetupPhase phase, + @PositiveOrZero long nextPollAfterMillis, + boolean exportAvailable) { + } + + /** Safe asynchronous operation view. */ + public record OperationResponse( + @NotBlank String operationId, + @NotNull SetupOperationState state, + @NotNull SetupPhase phase, + @NotNull Instant createdAt, + Instant startedAt, + Instant completedAt, + SetupErrorCode errorCode, + @PositiveOrZero long nextPollAfterMillis, + boolean exportAvailable) { + } + + /** Initial administrator input. */ + public record AdministratorRequest( + @NotBlank String username, + @NotBlank @JsonProperty(access = JsonProperty.Access.WRITE_ONLY) String password) { + + @Override + public String toString() { + return "AdministratorRequest[username=" + username + ", password=]"; + } + } + + /** Initial administrator result. */ + public record AdministratorResponse(@NotBlank String username, @NotNull SetupPhase phase) { + } + + /** Optional retention input. */ + public record RetentionConfiguration( + @Positive Integer metricsDays, + @Positive Integer logsDays, + @Positive Integer tracesDays) { + + public RetentionConfiguration { + requirePositiveIfPresent(metricsDays); + requirePositiveIfPresent(logsDays); + requirePositiveIfPresent(tracesDays); + } + } + + /** Optional setup input. */ + public record OptionsRequest( + @Valid PublicAccessConfiguration publicAccess, + @Valid RetentionConfiguration retention, + @Valid MailConfiguration mail) { + } + + /** Secret-free optional setup result. */ + public record OptionsResponse( + boolean publicAccessConfigured, + boolean serverOtlpHttpConfigured, + boolean serverOtlpGrpcConfigured, + boolean retentionConfigured, + boolean mailConfigured, + @NotNull SetupPhase phase) { + } + + /** Supported external configuration export. */ + public enum ExportFormat implements WireValue { + YAML("yaml"), + ENV("env"), + KUBERNETES_SECRET("kubernetes_secret"); + + private final String value; + + ExportFormat(String value) { + this.value = value; + } + + @Override + public String value() { + return value; + } + } + + /** Export input. */ + public record ExportRequest( + @NotNull ExportFormat format, + @NotNull @Valid ConfigurationRequest configuration) { + } + + /** Safe download metadata. Secret-bearing content is written only as a no-store attachment. */ + public record ExportResponse(@NotBlank String fileName, @NotBlank String mediaType) { + } + + /** Setup completion acknowledgement. */ + public record CompleteRequest( + @NotNull SetupPhase expectedPhase, + @NotNull List acknowledgedWarnings) { + + public CompleteRequest { + acknowledgedWarnings = List.copyOf(acknowledgedWarnings); + } + } + + /** Completed setup result. */ + public record CompleteResponse( + @NotNull SetupPhase phase, + @NotNull Instant completedAt, + @NotBlank String loginPath, + @NotBlank String username) { + + public CompleteResponse { + if (phase != SetupPhase.COMPLETE) { + throw new IllegalArgumentException("A complete response must report the complete phase"); + } + } + } + + private static int countPresent(Object... values) { + int count = 0; + for (Object value : values) { + if (value != null) { + count++; + } + } + return count; + } + + private static void requirePositiveIfPresent(Integer value) { + if (value != null && value <= 0) { + throw new IllegalArgumentException("Retention days must be positive when supplied"); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupHttpContract.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupHttpContract.java new file mode 100644 index 0000000000..0f95d89ca7 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupHttpContract.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import org.springframework.http.CacheControl; +import org.springframework.http.ResponseEntity; + +/** HTTP response policy shared by setup and deployment controllers. */ +public final class SetupHttpContract { + + private SetupHttpContract() { + } + + /** Creates a response builder that prevents setup and deployment data from being stored. */ + public static ResponseEntity.BodyBuilder noStore() { + return ResponseEntity.ok().cacheControl(CacheControl.noStore()); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupWorkflow.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupWorkflow.java new file mode 100644 index 0000000000..5e29892988 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupWorkflow.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.AdministratorRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.AdministratorResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.CompleteRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.CompleteResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OperationResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.StatusResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.UnlockRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.UnlockResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidateRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationResponse; + +/** Application boundary implemented by a later setup engine milestone. */ +public interface SetupWorkflow { + + StatusResponse status(); + + UnlockResponse unlock(UnlockRequest request); + + ValidationResponse validate(ValidateRequest request); + + ConfigurationResponse configure(ConfigurationRequest request); + + OperationResponse operation(String operationId); + + AdministratorResponse createAdministrator(AdministratorRequest request); + + OptionsResponse configureOptions(OptionsRequest request); + + ExportResponse prepareExport(ExportRequest request); + + CompleteResponse complete(CompleteRequest request); +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContractTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContractTest.java new file mode 100644 index 0000000000..64f667d930 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContractTest.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.lang.reflect.RecordComponent; +import java.util.Arrays; +import java.util.List; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.api.Test; + +/** Freezes authenticated deployment and H2 migration contracts. */ +class DeploymentApiContractTest { + + private final ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules(); + + @Test + void freezesDeploymentRoutesAndShapes() throws Exception { + assertEquals("/api/config/deployment", DeploymentApiContract.DEPLOYMENT_PATH); + assertEquals("/api/config/deployment/validate", DeploymentApiContract.VALIDATE_PATH); + assertEquals("/api/config/deployment/metadata-migrations", DeploymentApiContract.MIGRATION_PATH); + assertEquals("/api/config/deployment/metadata-migrations/{operationId}", + DeploymentApiContract.MIGRATION_OPERATION_PATH); + assertEquals("/api/config/deployment/metadata-migrations/{operationId}/activate", + DeploymentApiContract.ACTIVATE_PATH); + assertComponents(DeploymentApiContract.DeploymentView.class, "observedAt", "managementDatabase", + "telemetryStore", "applyMode", "maintenanceMode", "migrationAllowed"); + assertComponents(DeploymentApiContract.MetadataMigrationRequest.class, "target", "targetDatabase", "applyMode"); + assertComponents(DeploymentApiContract.MigrationView.class, "operationId", "state", "source", "target", + "phase", "createdAt", "startedAt", "completedAt", "tablesTotal", "tablesCopied", + "verificationState", "errorCode", "activationAvailable", "externalApplyRequired"); + assertComponents(DeploymentApiContract.ActivateMigrationRequest.class, "expectedState"); + assertWireValues(MigrationTarget.values(), "mysql", "postgresql"); + assertWireValues(VerificationState.values(), "pending", "running", "succeeded", "failed"); + assertEquals(DeploymentApiContract.MigrationView.class, + DeploymentWorkflow.class.getMethod( + "activate", String.class, DeploymentApiContract.ActivateMigrationRequest.class) + .getReturnType()); + } + + @Test + void migrationSourceIsFixedToH2AndTargetKindMustMatch() { + MetadataDatabaseConfiguration mysql = new MetadataDatabaseConfiguration( + MetadataDatabaseKind.MYSQL, "jdbc:mysql://db/hertzbeat", "user", "secret"); + assertEquals(mysql, new DeploymentApiContract.MetadataMigrationRequest( + MigrationTarget.MYSQL, mysql, ApplyMode.MANAGED_WRITE).targetDatabase()); + MetadataDatabaseConfiguration postgres = new MetadataDatabaseConfiguration( + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db/hertzbeat", "user", "secret"); + assertThrows(IllegalArgumentException.class, () -> new DeploymentApiContract.MetadataMigrationRequest( + MigrationTarget.MYSQL, postgres, ApplyMode.MANAGED_WRITE)); + MetadataDatabaseConfiguration h2 = new MetadataDatabaseConfiguration( + MetadataDatabaseKind.H2, "jdbc:h2:file:./data/hertzbeat", "sa", "secret"); + assertThrows(IllegalArgumentException.class, () -> new DeploymentApiContract.MetadataMigrationRequest( + MigrationTarget.POSTGRESQL, h2, ApplyMode.EXTERNAL_APPLY)); + } + + @Test + void migrationRequestDoesNotSerializeOrRenderTargetPassword() throws Exception { + String secret = "migration-contract-secret"; + MetadataDatabaseConfiguration mysql = new MetadataDatabaseConfiguration( + MetadataDatabaseKind.MYSQL, "jdbc:mysql://db/hertzbeat", "user", secret); + DeploymentApiContract.MetadataMigrationRequest request = new DeploymentApiContract.MetadataMigrationRequest( + MigrationTarget.MYSQL, mysql, ApplyMode.MANAGED_WRITE); + assertFalse(objectMapper.writeValueAsString(request).contains(secret)); + assertFalse(request.toString().contains(secret)); + } + + private void assertComponents(Class type, String... names) { + assertEquals(List.of(names), Arrays.stream(type.getRecordComponents()).map(RecordComponent::getName).toList()); + } + + private void assertWireValues(Enum[] values, String... expected) throws Exception { + assertEquals(List.of(expected), Arrays.stream(values).map(this::wireValue).toList()); + } + + private String wireValue(Enum value) { + try { + return objectMapper.writeValueAsString(value).replace("\"", ""); + } catch (Exception exception) { + throw new IllegalStateException(exception); + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java new file mode 100644 index 0000000000..8728f4fbf7 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.lang.reflect.RecordComponent; +import java.time.Instant; +import java.util.Arrays; +import java.util.List; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.AdministratorRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigSource; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.PublicAccessConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupAccess; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.UnlockRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidateRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationSection; +import org.junit.jupiter.api.Test; + +/** Freezes the unversioned first-install setup contract. */ +class SetupApiContractTest { + + private static final String SECRET = "contract-secret"; + + private final ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules(); + + @Test + void freezesRoutesAndWireEnums() throws Exception { + assertEquals("/api/setup/status", SetupApiContract.STATUS_PATH); + assertEquals("/api/setup/unlock", SetupApiContract.UNLOCK_PATH); + assertEquals("/api/setup/validate", SetupApiContract.VALIDATE_PATH); + assertEquals("/api/setup/configuration", SetupApiContract.CONFIGURATION_PATH); + assertEquals("/api/setup/operations/{operationId}", SetupApiContract.OPERATION_PATH); + assertEquals("/api/setup/administrator", SetupApiContract.ADMINISTRATOR_PATH); + assertEquals("/api/setup/options", SetupApiContract.OPTIONS_PATH); + assertEquals("/api/setup/export", SetupApiContract.EXPORT_PATH); + assertEquals("/api/setup/complete", SetupApiContract.COMPLETE_PATH); + assertWireValues(SetupPhase.values(), "configuration_required", "external_apply_required", + "application_starting", "administrator_required", "optional_configuration", "complete", + "recovery_required", "migration_in_progress"); + assertWireValues(ConfigSource.values(), "built_in_default", "ui_managed", "external_file", "environment", + "system_property", "command_line"); + assertWireValues(ApplyMode.values(), "managed_write", "external_apply"); + assertWireValues(SetupAccess.values(), "local", "locked", "unlocked"); + assertWireValues(SetupOperationState.values(), "pending", "running", "awaiting_external_apply", + "awaiting_restart", "succeeded", "failed", "rolled_back"); + assertWireValues(MetadataDatabaseKind.values(), "h2", "mysql", "postgresql"); + assertWireValues(TelemetryStoreKind.values(), "greptime"); + assertWireValues(ValidationSection.values(), "metadata_database", "telemetry_store", "public_access", "mail"); + assertWireValues(MailSecurity.values(), "none", "starttls", "tls"); + assertWireValues(SetupApiContract.ExportFormat.values(), "yaml", "env", "kubernetes_secret"); + assertWireValues(SetupApiContract.SetupWarningCode.values(), "external_apply_required", "restart_required", + "public_address_plaintext", "mail_security_none"); + } + + @Test + void freezesSafeStatusAndMutationShapes() { + assertComponents(SetupApiContract.StatusResponse.class, "phase", "observedAt", "access", "applyMode", + "writableManagedConfig", "operationId", "errorCode", "managementDatabase", "telemetryStore", + "administratorConfigured", "optional"); + assertComponents(SetupApiContract.ManagementDatabaseSummary.class, "kind", "configured", "source", + "restartRequired"); + assertComponents(SetupApiContract.TelemetryStoreSummary.class, "kind", "configured", "source", + "restartRequired"); + assertComponents(SetupApiContract.OptionalConfigurationSummary.class, "publicAccessConfigured", + "serverOtlpHttpConfigured", "serverOtlpGrpcConfigured", "retentionConfigured", "mailConfigured"); + assertComponents(SetupApiContract.UnlockRequest.class, "code"); + assertComponents(SetupApiContract.UnlockResponse.class, "access", "expiresAt"); + assertComponents(SetupApiContract.ValidateRequest.class, "section", "managementDatabase", "telemetryStore", + "publicAccess", "mail"); + assertComponents(SetupApiContract.TelemetryStoreConfiguration.class, "kind", "grpcEndpoints", "httpEndpoint", + "database", "username", "password"); + assertComponents(SetupApiContract.ValidationResponse.class, "valid", "observedAt", "errorCode", "warnings"); + assertComponents(SetupApiContract.ConfigurationRequest.class, "expectedPhase", "applyMode", + "managementDatabase", "telemetryStore"); + assertComponents(SetupApiContract.ConfigurationResponse.class, "operationId", "state", "phase", + "nextPollAfterMillis", "exportAvailable"); + assertComponents(SetupApiContract.OperationResponse.class, "operationId", "state", "phase", "createdAt", + "startedAt", "completedAt", "errorCode", "nextPollAfterMillis", "exportAvailable"); + assertComponents(SetupApiContract.AdministratorRequest.class, "username", "password"); + assertComponents(SetupApiContract.AdministratorResponse.class, "username", "phase"); + assertComponents(SetupApiContract.OptionsRequest.class, "publicAccess", "retention", "mail"); + assertComponents(SetupApiContract.RetentionConfiguration.class, "metricsDays", "logsDays", "tracesDays"); + assertComponents(SetupApiContract.OptionsResponse.class, "publicAccessConfigured", "serverOtlpHttpConfigured", + "serverOtlpGrpcConfigured", "retentionConfigured", "mailConfigured", "phase"); + assertComponents(SetupApiContract.ExportRequest.class, "format", "configuration"); + assertComponents(SetupApiContract.ExportResponse.class, "fileName", "mediaType"); + assertComponents(SetupApiContract.CompleteRequest.class, "expectedPhase", "acknowledgedWarnings"); + assertComponents(SetupApiContract.CompleteResponse.class, "phase", "completedAt", "loginPath", "username"); + } + + @Test + void secretInputsAreWriteOnlyAndSafeToRender() throws Exception { + MetadataDatabaseConfiguration metadata = new MetadataDatabaseConfiguration( + MetadataDatabaseKind.MYSQL, "jdbc:mysql://db/hertzbeat", "user", SECRET); + TelemetryStoreConfiguration telemetry = new TelemetryStoreConfiguration( + TelemetryStoreKind.GREPTIME, "greptime:4001", "http://greptime:4000", "public", "user", SECRET); + MailConfiguration mail = new MailConfiguration( + "mail.example.test", 587, MailSecurity.STARTTLS, "user", SECRET, "ops@example.test"); + SetupApiContract.ConfigurationRequest configuration = new SetupApiContract.ConfigurationRequest( + SetupPhase.CONFIGURATION_REQUIRED, ApplyMode.MANAGED_WRITE, metadata, telemetry); + List requests = List.of( + new UnlockRequest(SECRET), + new AdministratorRequest("admin", SECRET), + metadata, + telemetry, + mail, + new ValidateRequest(ValidationSection.METADATA_DATABASE, metadata, null, null, null), + configuration, + new SetupApiContract.OptionsRequest(null, null, mail), + new SetupApiContract.ExportRequest(SetupApiContract.ExportFormat.YAML, configuration)); + for (Object request : requests) { + assertFalse(objectMapper.writeValueAsString(request).contains(SECRET)); + assertFalse(request.toString().contains(SECRET)); + } + UnlockRequest decoded = objectMapper.readValue("{\"code\":\"" + SECRET + "\"}", UnlockRequest.class); + assertEquals(SECRET, decoded.code()); + } + + @Test + void validateRequestRequiresExactlyOneMatchingSection() { + MetadataDatabaseConfiguration metadata = new MetadataDatabaseConfiguration( + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db/hertzbeat", "user", SECRET); + assertEquals(metadata, new ValidateRequest( + ValidationSection.METADATA_DATABASE, metadata, null, null, null).managementDatabase()); + assertThrows(IllegalArgumentException.class, + () -> new ValidateRequest(ValidationSection.METADATA_DATABASE, null, null, null, null)); + assertThrows(IllegalArgumentException.class, () -> new ValidateRequest( + ValidationSection.METADATA_DATABASE, metadata, null, + new PublicAccessConfiguration("http://localhost:1157", null, null), null)); + assertThrows(IllegalArgumentException.class, () -> new ValidateRequest( + ValidationSection.MAIL, metadata, null, null, null)); + } + + @Test + void freezesStableSafeErrorCodes() throws Exception { + assertWireValues(SetupErrorCode.values(), "setup_complete", "setup_locked", "setup_code_invalid", + "setup_code_expired", "setup_rate_limited", "config_read_only", "config_write_failed", + "config_recovery_required", "metadata_connection_failed", "metadata_kind_unsupported", + "metadata_schema_mismatch", "metadata_insufficient_privileges", "telemetry_connection_failed", + "public_address_invalid", "mail_connection_failed", "administrator_already_configured", + "administrator_username_invalid", "operation_not_found", "operation_conflict", + "migration_source_unsupported", "migration_target_not_empty", "migration_multi_node_unsupported", + "migration_copy_failed", "migration_verification_failed", "migration_activation_failed", + "restart_failed"); + } + + @Test + void safeStatusContainsNoConnectionDetails() throws Exception { + SetupApiContract.StatusResponse response = new SetupApiContract.StatusResponse( + SetupPhase.CONFIGURATION_REQUIRED, + Instant.parse("2026-08-07T00:00:00Z"), + SetupAccess.LOCAL, + ApplyMode.MANAGED_WRITE, + true, + null, + null, + new SetupApiContract.ManagementDatabaseSummary( + MetadataDatabaseKind.H2, true, ConfigSource.BUILT_IN_DEFAULT, false), + new SetupApiContract.TelemetryStoreSummary( + TelemetryStoreKind.GREPTIME, false, ConfigSource.BUILT_IN_DEFAULT, false), + false, + new SetupApiContract.OptionalConfigurationSummary(false, false, false, false, false)); + String json = objectMapper.writeValueAsString(response); + assertFalse(json.contains("jdbc")); + assertFalse(json.contains("username")); + assertFalse(json.contains("password")); + assertFalse(json.contains("exception")); + assertFalse(json.contains("fingerprint")); + } + + private void assertComponents(Class type, String... names) { + assertEquals(List.of(names), Arrays.stream(type.getRecordComponents()).map(RecordComponent::getName).toList()); + } + + private void assertWireValues(Enum[] values, String... expected) throws Exception { + assertEquals(List.of(expected), Arrays.stream(values).map(this::wireValue).toList()); + } + + private String wireValue(Enum value) { + try { + return objectMapper.writeValueAsString(value).replace("\"", ""); + } catch (Exception exception) { + throw new IllegalStateException(exception); + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupHttpContractTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupHttpContractTest.java new file mode 100644 index 0000000000..6658cf9a11 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupHttpContractTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import org.junit.jupiter.api.Test; +import org.springframework.http.ResponseEntity; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RestController; + +/** Guards the no-store HTTP boundary independently of controller implementation. */ +class SetupHttpContractTest { + + @Test + void setupAndDeploymentResponsesAreNoStore() throws Exception { + MockMvc mvc = MockMvcBuilders.standaloneSetup(new ContractController()).build(); + mvc.perform(get(SetupApiContract.STATUS_PATH)) + .andExpect(status().isOk()) + .andExpect(header().string("Cache-Control", "no-store")); + mvc.perform(get(DeploymentApiContract.DEPLOYMENT_PATH)) + .andExpect(status().isOk()) + .andExpect(header().string("Cache-Control", "no-store")); + mvc.perform(post(SetupApiContract.COMPLETE_PATH)) + .andExpect(status().isOk()) + .andExpect(header().string("Cache-Control", "no-store")); + } + + @RestController + static class ContractController { + + @GetMapping(SetupApiContract.STATUS_PATH) + ResponseEntity status() { + return SetupHttpContract.noStore().build(); + } + + @GetMapping(DeploymentApiContract.DEPLOYMENT_PATH) + ResponseEntity deployment() { + return SetupHttpContract.noStore().build(); + } + + @PostMapping(SetupApiContract.COMPLETE_PATH) + ResponseEntity complete() { + return SetupHttpContract.noStore().build(); + } + } +} From 29ac8119168ae411e73ad477018ec70c45799beb Mon Sep 17 00:00:00 2001 From: Logic Date: Sat, 8 Aug 2026 03:26:28 +0800 Subject: [PATCH 08/71] Add setup-gated application runtime --- .../apache/hertzbeat/ai/config/LlmConfig.java | 4 +- .../ai/config/LlmConfigInitializer.java | 39 ++++ .../ai/schedule/SopScheduleExecutor.java | 2 + .../periodic/PeriodicAlertRuleScheduler.java | 130 +++++++------ .../PeriodicAlertRuleSchedulerLifecycle.java | 45 +++++ .../WindowedLogRealTimeAlertCalculator.java | 2 + .../calculate/realtime/window/LogWorker.java | 4 +- .../realtime/window/TimeService.java | 2 + .../realtime/window/WindowAggregator.java | 2 + .../alert/reduce/AlarmGroupReduce.java | 78 +++++--- .../alert/reduce/AlarmInhibitReduce.java | 79 +++++--- .../alert/reduce/AlarmReduceLifecycle.java | 42 ++++ .../service/impl/NoticeConfigServiceImpl.java | 6 +- .../impl/NoticeTemplateInitializer.java | 42 ++++ .../PeriodicAlertRuleSchedulerTest.java | 48 ++++- .../alert/reduce/AlarmGroupReduceTest.java | 55 +++++- .../alert/reduce/AlarmInhibitReduceTest.java | 58 +++++- .../common/runtime/BusinessRuntimeGate.java | 42 ++++ .../hertzbeat/common/runtime/RuntimeMode.java | 53 +++++ .../runtime/BusinessRuntimeGateTest.java | 39 ++++ .../runtime/BusinessRuntimeConfiguration.java | 34 ++++ .../ConditionalOnNormalBusinessRuntime.java | 31 +++ .../NormalBusinessRuntimeConditionTest.java | 61 ++++++ .../hertzbeat/grafana/config/GrafanaInit.java | 2 + .../hertzbeat/log/notice/LogSseManager.java | 51 +++-- .../log/notice/LogSseManagerLifecycle.java | 39 ++++ .../log/notice/LogSseManagerTest.java | 14 +- .../component/sd/ServiceDiscoveryWorker.java | 2 + .../component/status/CalculateStatus.java | 137 +++++++++---- .../status/CalculateStatusLifecycle.java | 39 ++++ .../manager/config/ConfigInitializer.java | 2 + .../manager/scheduler/SchedulerInit.java | 2 + .../manager/scheduler/netty/ManageServer.java | 182 +++++++++++------- .../netty/ManageServerLifecycle.java | 50 +++++ .../LocalTopologyDemoRelationSeeder.java | 2 + .../manager/service/impl/AppServiceImpl.java | 8 +- .../ManagerBusinessRuntimeInitializer.java | 56 ++++++ .../impl/ObjectStoreConfigServiceImpl.java | 7 +- .../impl/PluginParameterServiceImpl.java | 2 - .../service/impl/PluginServiceImpl.java | 7 +- .../manager/setup/api/SetupApiContract.java | 1 + .../SetupRuntimeAccessConfiguration.java | 45 +++++ .../runtime/SetupRuntimeAccessFilter.java | 82 ++++++++ .../setup/runtime/SetupRuntimeTransition.java | 30 +++ .../component/status/CalculateStatusTest.java | 84 +++++++- .../MonitorDefinitionCommandPortTest.java | 2 +- .../scheduler/netty/ManageServerTest.java | 70 +++++++ .../manager/service/AppServiceTest.java | 2 +- .../service/ObjectStoreConfigServiceTest.java | 2 +- .../setup/api/SetupApiContractTest.java | 2 +- .../runtime/SetupRuntimeAccessFilterTest.java | 116 +++++++++++ ...OpenTelemetryLogbackAppenderInstaller.java | 2 + .../config/OtlpGrpcServerConfig.java | 2 + .../forwarder/GreptimeApmFlowInitializer.java | 2 + .../GreptimeLogPipelineInitializer.java | 2 + .../GreptimeTraceTableInitializer.java | 2 + .../observability/logs/sse/LogSseManager.java | 22 ++- .../logs/sse/LogSseManagerLifecycle.java | 39 ++++ .../logs/sse/LogSseManagerTest.java | 1 + ...OpenTelemetryLogbackAppenderInstaller.java | 4 +- .../bootstrap/SetupOnlyApplication.java | 50 +++++ .../startup/HertzBeatApplication.java | 20 +- .../runtime/HertzBeatStartupCoordinator.java | 96 +++++++++ .../runtime/RunningApplicationContext.java | 31 +++ .../runtime/SpringStartupContextLauncher.java | 70 +++++++ .../runtime/StartupContextLauncher.java | 28 +++ .../startup/runtime/StartupDecision.java | 41 ++++ .../startup/runtime/StartupDecisionProbe.java | 25 +++ .../runtime/StartupModePropertyProbe.java | 63 ++++++ .../HertzBeatStartupCoordinatorTest.java | 150 +++++++++++++++ .../runtime/StartupModePropertyProbeTest.java | 67 +++++++ .../StartupRuntimeBoundaryContextTest.java | 115 +++++++++++ .../GreptimeTtlApplicationReadyListener.java | 2 + .../history/tsdb/doris/DorisDataStorage.java | 2 + .../duckdb/DuckdbDatabaseDataStorage.java | 2 + .../tsdb/greptime/GreptimeDbDataStorage.java | 2 + .../tsdb/influxdb/InfluxdbDataStorage.java | 2 + .../history/tsdb/iotdb/IotDbDataStorage.java | 2 + .../tsdb/questdb/QuestdbDataStorage.java | 4 +- .../tsdb/tdengine/TdEngineDataStorage.java | 2 + .../vm/VictoriaMetricsClusterDataStorage.java | 2 + .../tsdb/vm/VictoriaMetricsDataStorage.java | 2 + 82 files changed, 2505 insertions(+), 285 deletions(-) create mode 100644 hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/config/LlmConfigInitializer.java create mode 100644 hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerLifecycle.java create mode 100644 hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmReduceLifecycle.java create mode 100644 hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/service/impl/NoticeTemplateInitializer.java create mode 100644 hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/runtime/BusinessRuntimeGate.java create mode 100644 hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/runtime/RuntimeMode.java create mode 100644 hertzbeat-common-core/src/test/java/org/apache/hertzbeat/common/runtime/BusinessRuntimeGateTest.java create mode 100644 hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/runtime/BusinessRuntimeConfiguration.java create mode 100644 hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/runtime/ConditionalOnNormalBusinessRuntime.java create mode 100644 hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/runtime/NormalBusinessRuntimeConditionTest.java create mode 100644 hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseManagerLifecycle.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatusLifecycle.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServerLifecycle.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/ManagerBusinessRuntimeInitializer.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeAccessConfiguration.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeAccessFilter.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransition.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeAccessFilterTest.java create mode 100644 hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/logs/sse/LogSseManagerLifecycle.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/bootstrap/SetupOnlyApplication.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/RunningApplicationContext.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/SpringStartupContextLauncher.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupContextLauncher.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupDecision.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupDecisionProbe.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbe.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinatorTest.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbeTest.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupRuntimeBoundaryContextTest.java diff --git a/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/config/LlmConfig.java b/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/config/LlmConfig.java index 5e77998c53..9601b957c5 100644 --- a/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/config/LlmConfig.java +++ b/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/config/LlmConfig.java @@ -19,7 +19,6 @@ package org.apache.hertzbeat.ai.config; import com.openai.client.OpenAIClient; -import jakarta.annotation.PostConstruct; import java.time.Duration; import java.util.Map; import lombok.extern.slf4j.Slf4j; @@ -58,8 +57,7 @@ public class LlmConfig { this.applicationContext = applicationContext; } - @PostConstruct - public void registerInitialChatClient() { + void registerInitialChatClient() { registerChatClient(); } diff --git a/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/config/LlmConfigInitializer.java b/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/config/LlmConfigInitializer.java new file mode 100644 index 0000000000..dddd150e3c --- /dev/null +++ b/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/config/LlmConfigInitializer.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.ai.config; + +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; +import org.springframework.boot.CommandLineRunner; +import org.springframework.stereotype.Component; + +/** Registers the configured chat client only after the business runtime opens. */ +@Component +@ConditionalOnNormalBusinessRuntime +public final class LlmConfigInitializer implements CommandLineRunner { + + private final LlmConfig llmConfig; + + public LlmConfigInitializer(LlmConfig llmConfig) { + this.llmConfig = llmConfig; + } + + @Override + public void run(String... args) { + llmConfig.registerInitialChatClient(); + } +} diff --git a/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/schedule/SopScheduleExecutor.java b/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/schedule/SopScheduleExecutor.java index 6e346f98c0..a5ae414614 100644 --- a/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/schedule/SopScheduleExecutor.java +++ b/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/schedule/SopScheduleExecutor.java @@ -30,6 +30,7 @@ import org.apache.hertzbeat.ai.sop.registry.SkillRegistry; import org.apache.hertzbeat.ai.utils.SopMessageUtil; import org.apache.hertzbeat.common.entity.ai.ChatMessage; import org.apache.hertzbeat.common.entity.ai.SopSchedule; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.apache.hertzbeat.common.util.JsonUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; @@ -43,6 +44,7 @@ import org.springframework.stereotype.Component; */ @Slf4j @Component +@ConditionalOnNormalBusinessRuntime @EnableScheduling public class SopScheduleExecutor { diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java index 5182fb64ab..e3c4850ec0 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java @@ -20,8 +20,8 @@ package org.apache.hertzbeat.alert.calculate.periodic; import static org.apache.hertzbeat.common.constants.CommonConstants.LOG_ALERT_THRESHOLD_TYPE_PERIODIC; import static org.apache.hertzbeat.common.constants.CommonConstants.METRIC_ALERT_THRESHOLD_TYPE_PERIODIC; import static org.apache.hertzbeat.common.constants.CommonConstants.TRACE_ALERT_THRESHOLD_TYPE_PERIODIC; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -37,27 +37,26 @@ import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.alert.dao.AlertDefineDao; import org.apache.hertzbeat.common.config.VirtualThreadProperties; -import org.springframework.beans.factory.DisposableBean; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.CommandLineRunner; -import org.springframework.stereotype.Component; import org.apache.hertzbeat.common.entity.alerter.AlertDefine; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; /** * Periodic Alert Rule Scheduler */ @Slf4j @Component -public class PeriodicAlertRuleScheduler implements CommandLineRunner, DisposableBean { +public class PeriodicAlertRuleScheduler { private final MetricsPeriodicAlertCalculator metricsCalculator; private final LogPeriodicAlertCalculator logCalculator; private final TracePeriodicAlertCalculator traceCalculator; private final AlertDefineDao alertDefineDao; - private final ScheduledExecutorService scheduledExecutor; - private final ExecutorService periodicExecutor; - private final Semaphore periodicPermits; - private final boolean virtualThreadsEnabled; + private ScheduledExecutorService scheduledExecutor; + private ExecutorService periodicExecutor; + private Semaphore periodicPermits; + private final VirtualThreadProperties virtualThreadProperties; + private boolean virtualThreadsEnabled; private final Map scheduledTasks; @Autowired @@ -70,10 +69,15 @@ public class PeriodicAlertRuleScheduler implements CommandLineRunner, Disposable this.logCalculator = logCalculator; this.traceCalculator = traceCalculator; this.alertDefineDao = alertDefineDao; - Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { - log.error("Scheduled periodic alert threshold has uncaughtException."); - log.error(throwable.getMessage(), throwable); - }; + this.virtualThreadProperties = virtualThreadProperties == null + ? VirtualThreadProperties.defaults() : virtualThreadProperties; + this.scheduledTasks = new ConcurrentHashMap<>(); + } + + synchronized void start() { + if (scheduledExecutor != null) { + return; + } ThreadFactory threadFactory = new ThreadFactoryBuilder() .setUncaughtExceptionHandler((thread, throwable) -> { log.error("Scheduled periodic alert threshold has uncaughtException."); @@ -82,49 +86,18 @@ public class PeriodicAlertRuleScheduler implements CommandLineRunner, Disposable .setDaemon(true) .setNameFormat("periodic-alert-threshold-worker-%d") .build(); - this.scheduledExecutor = Executors.newScheduledThreadPool(10, threadFactory); - VirtualThreadProperties properties = - virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; - this.virtualThreadsEnabled = properties.enabled(); - int maxConcurrentPeriodicTasks = Math.max(1, properties.alerter().periodicMaxConcurrentJobs()); + scheduledExecutor = Executors.newScheduledThreadPool(10, threadFactory); + virtualThreadsEnabled = virtualThreadProperties.enabled(); + int maxConcurrentPeriodicTasks = Math.max( + 1, virtualThreadProperties.alerter().periodicMaxConcurrentJobs()); this.periodicExecutor = virtualThreadsEnabled ? Executors.newThreadPerTaskExecutor(Thread.ofVirtual() .name("periodic-alert-task-", 0) - .uncaughtExceptionHandler(handler) + .uncaughtExceptionHandler((thread, throwable) -> + log.error("Periodic alert task failed", throwable)) .factory()) : null; this.periodicPermits = virtualThreadsEnabled ? new Semaphore(maxConcurrentPeriodicTasks) : null; - this.scheduledTasks = new ConcurrentHashMap<>(); - } - - public void cancelSchedule(Long ruleId) { - if (ruleId == null) { - return; - } - ScheduledTaskState state = scheduledTasks.remove(ruleId); - if (state != null) { - state.cancel(); - } - } - - public void updateSchedule(AlertDefine rule) { - if (rule == null || rule.getId() == null) { - log.error("Alert rule is null or rule id is null."); - return; - } - cancelSchedule(rule.getId()); - if (isPeriodicRule(rule.getType())) { - ScheduledTaskState state = new ScheduledTaskState(rule); - ScheduledFuture future = scheduledExecutor.scheduleAtFixedRate( - virtualThreadsEnabled ? state::trigger : () -> executeRule(rule), - 0, rule.getPeriod(), TimeUnit.SECONDS); - state.setScheduledFuture(future); - scheduledTasks.put(rule.getId(), state); - } - } - - @Override - public void run(String... args) throws Exception { log.info("Starting periodic alert rule scheduler..."); List metricsPeriodicRules = alertDefineDao.findAlertDefinesByTypeAndEnableTrue(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC); List logPeriodicRules = alertDefineDao.findAlertDefinesByTypeAndEnableTrue(LOG_ALERT_THRESHOLD_TYPE_PERIODIC); @@ -139,13 +112,50 @@ public class PeriodicAlertRuleScheduler implements CommandLineRunner, Disposable } } - @Override - public void destroy() { + synchronized void stop() { scheduledTasks.values().forEach(ScheduledTaskState::cancel); scheduledTasks.clear(); - scheduledExecutor.shutdownNow(); + if (scheduledExecutor != null) { + scheduledExecutor.shutdownNow(); + scheduledExecutor = null; + } if (periodicExecutor != null) { periodicExecutor.shutdownNow(); + periodicExecutor = null; + } + periodicPermits = null; + } + + public synchronized void cancelSchedule(Long ruleId) { + if (ruleId == null || scheduledExecutor == null) { + return; + } + ScheduledTaskState state = scheduledTasks.remove(ruleId); + if (state != null) { + state.cancel(); + } + } + + public synchronized void updateSchedule(AlertDefine rule) { + if (rule == null || rule.getId() == null) { + log.error("Alert rule is null or rule id is null."); + return; + } + if (scheduledExecutor == null) { + return; + } + cancelSchedule(rule.getId()); + if (isPeriodicRule(rule.getType())) { + ScheduledExecutorService currentScheduledExecutor = scheduledExecutor; + ExecutorService currentPeriodicExecutor = periodicExecutor; + Semaphore currentPeriodicPermits = periodicPermits; + ScheduledTaskState state = new ScheduledTaskState( + rule, currentPeriodicExecutor, currentPeriodicPermits); + ScheduledFuture future = currentScheduledExecutor.scheduleAtFixedRate( + virtualThreadsEnabled ? state::trigger : () -> executeRule(rule), + 0, rule.getPeriod(), TimeUnit.SECONDS); + state.setScheduledFuture(future); + scheduledTasks.put(rule.getId(), state); } } @@ -168,14 +178,18 @@ public class PeriodicAlertRuleScheduler implements CommandLineRunner, Disposable private final class ScheduledTaskState { private final AlertDefine rule; + private final ExecutorService taskExecutor; + private final Semaphore taskPermits; private ScheduledFuture scheduledFuture; private Future runningFuture; private boolean running; private boolean pending; private boolean cancelled; - private ScheduledTaskState(AlertDefine rule) { + private ScheduledTaskState(AlertDefine rule, ExecutorService taskExecutor, Semaphore taskPermits) { this.rule = rule; + this.taskExecutor = taskExecutor; + this.taskPermits = taskPermits; } private synchronized void setScheduledFuture(ScheduledFuture scheduledFuture) { @@ -209,10 +223,10 @@ public class PeriodicAlertRuleScheduler implements CommandLineRunner, Disposable private void submitLocked() { try { - runningFuture = periodicExecutor.submit(() -> { + runningFuture = taskExecutor.submit(() -> { boolean permitAcquired = false; try { - periodicPermits.acquire(); + taskPermits.acquire(); permitAcquired = true; if (!Thread.currentThread().isInterrupted()) { executeRule(rule); @@ -223,7 +237,7 @@ public class PeriodicAlertRuleScheduler implements CommandLineRunner, Disposable log.error("Periodic alert rule {} execution error: {}", rule.getName(), e.getMessage(), e); } finally { if (permitAcquired) { - periodicPermits.release(); + taskPermits.release(); } onComplete(); } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerLifecycle.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerLifecycle.java new file mode 100644 index 0000000000..8a583d9ff9 --- /dev/null +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerLifecycle.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.alert.calculate.periodic; + +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.boot.CommandLineRunner; +import org.springframework.stereotype.Component; + +/** Owns scheduler threads and database startup reads only in normal business runtime. */ +@Component +@ConditionalOnNormalBusinessRuntime +public final class PeriodicAlertRuleSchedulerLifecycle implements CommandLineRunner, DisposableBean { + + private final PeriodicAlertRuleScheduler scheduler; + + public PeriodicAlertRuleSchedulerLifecycle(PeriodicAlertRuleScheduler scheduler) { + this.scheduler = scheduler; + } + + @Override + public void run(String... args) { + scheduler.start(); + } + + @Override + public void destroy() { + scheduler.stop(); + } +} diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/WindowedLogRealTimeAlertCalculator.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/WindowedLogRealTimeAlertCalculator.java index 5a71d5fb27..13b73e7e37 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/WindowedLogRealTimeAlertCalculator.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/WindowedLogRealTimeAlertCalculator.java @@ -25,6 +25,7 @@ import org.apache.hertzbeat.alert.calculate.realtime.window.LogWorker; import org.apache.hertzbeat.alert.calculate.realtime.window.TimeService; import org.apache.hertzbeat.common.entity.log.LogEntry; import org.apache.hertzbeat.common.queue.CommonDataQueue; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.apache.hertzbeat.common.support.exception.CommonDataQueueUnknownException; import org.apache.hertzbeat.common.util.BackoffUtils; import org.apache.hertzbeat.common.util.ExponentialBackoff; @@ -44,6 +45,7 @@ import java.util.concurrent.TimeUnit; * 4. Distributing logs to workers */ @Component +@ConditionalOnNormalBusinessRuntime @Slf4j public class WindowedLogRealTimeAlertCalculator implements Runnable { diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/LogWorker.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/LogWorker.java index 16f25c62b4..8d0ec0559d 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/LogWorker.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/LogWorker.java @@ -23,6 +23,7 @@ import org.apache.hertzbeat.alert.calculate.JexlExprCalculator; import org.apache.hertzbeat.alert.service.AlertDefineService; import org.apache.hertzbeat.common.entity.alerter.AlertDefine; import org.apache.hertzbeat.common.entity.log.LogEntry; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -38,6 +39,7 @@ import java.util.Map; */ @Slf4j @Component +@ConditionalOnNormalBusinessRuntime public class LogWorker { private static final String LOG_PREFIX = "log"; @@ -120,4 +122,4 @@ public class LogWorker { } return System.currentTimeMillis(); } -} \ No newline at end of file +} diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/TimeService.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/TimeService.java index d7d4d82bf7..341ed40413 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/TimeService.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/TimeService.java @@ -24,6 +24,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.springframework.stereotype.Component; import jakarta.annotation.PreDestroy; @@ -42,6 +43,7 @@ import java.util.concurrent.atomic.AtomicLong; * 3. Broadcasting watermarks to all subscribers (WindowAggregator) */ @Component +@ConditionalOnNormalBusinessRuntime @Slf4j public class TimeService { diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/WindowAggregator.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/WindowAggregator.java index 068af89c3b..22852000e0 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/WindowAggregator.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/WindowAggregator.java @@ -24,6 +24,7 @@ import lombok.Data; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.common.entity.alerter.AlertDefine; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.springframework.stereotype.Component; import jakarta.annotation.PreDestroy; @@ -49,6 +50,7 @@ import java.util.concurrent.TimeUnit; * 4. Sending closed windows to AlarmEvaluator */ @Component +@ConditionalOnNormalBusinessRuntime @Slf4j public class WindowAggregator implements TimeService.WatermarkListener, Runnable { diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java index 693658b805..48040dc402 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java @@ -93,57 +93,70 @@ public class AlarmGroupReduce implements DisposableBean { */ private final Map groupCacheMap; - private final ScheduledExecutorService scheduledExecutor; + private final AlertGroupConvergeDao alertGroupConvergeDao; + private final VirtualThreadProperties virtualThreadProperties; + private ScheduledExecutorService scheduledExecutor; - private final ExecutorService workerExecutor; + private ExecutorService workerExecutor; - private final ScheduledDispatchTask checkTask; + private ScheduledDispatchTask checkTask; public AlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao) { - this(alarmInhibitReduce, alertGroupConvergeDao, VirtualThreadProperties.defaults(), true); + this(alarmInhibitReduce, alertGroupConvergeDao, VirtualThreadProperties.defaults()); } @Autowired public AlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao, VirtualThreadProperties virtualThreadProperties) { - this(alarmInhibitReduce, alertGroupConvergeDao, virtualThreadProperties, true); - } - - AlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao, - VirtualThreadProperties virtualThreadProperties, boolean autoStart) { this.alarmInhibitReduce = alarmInhibitReduce; this.groupDefines = new ConcurrentHashMap<>(8); this.groupCacheMap = new ConcurrentHashMap<>(8); - VirtualThreadProperties properties = - virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; - this.scheduledExecutor = createScheduler(); - this.workerExecutor = createVirtualExecutor(properties); - this.checkTask = new ScheduledDispatchTask(workerExecutor, this::runCheckAndSendGroups); - List groupConverges = alertGroupConvergeDao.findAlertGroupConvergesByEnableIsTrue(); - refreshGroupDefines(groupConverges); - if (autoStart) { - startCheckAndSendGroups(); - } + this.alertGroupConvergeDao = alertGroupConvergeDao; + this.virtualThreadProperties = virtualThreadProperties == null + ? VirtualThreadProperties.defaults() : virtualThreadProperties; } - private void startCheckAndSendGroups() { - scheduledExecutor.scheduleAtFixedRate(this::dispatchCheckAndSendGroups, 10000, CHECK_INTERVAL, + synchronized void start() { + if (scheduledExecutor != null) { + return; + } + scheduledExecutor = createScheduler(); + workerExecutor = createVirtualExecutor(virtualThreadProperties); + ScheduledDispatchTask currentCheckTask = + new ScheduledDispatchTask(workerExecutor, this::runCheckAndSendGroups); + checkTask = currentCheckTask; + refreshGroupDefines(alertGroupConvergeDao.findAlertGroupConvergesByEnableIsTrue()); + startCheckAndSendGroups(currentCheckTask); + } + + private void startCheckAndSendGroups(ScheduledDispatchTask currentCheckTask) { + scheduledExecutor.scheduleAtFixedRate(currentCheckTask::dispatch, 10000, CHECK_INTERVAL, TimeUnit.MILLISECONDS); } - void dispatchCheckAndSendGroups() { - checkTask.dispatch(); + synchronized void dispatchCheckAndSendGroups() { + if (checkTask != null) { + checkTask.dispatch(); + } } void beforeCheckAndSendGroupsRun() { } @Override - public void destroy() { - scheduledExecutor.shutdownNow(); + public synchronized void destroy() { + if (checkTask != null) { + checkTask.cancel(); + } + if (scheduledExecutor != null) { + scheduledExecutor.shutdownNow(); + scheduledExecutor = null; + } if (workerExecutor != null) { workerExecutor.shutdownNow(); + workerExecutor = null; } + checkTask = null; } private ScheduledExecutorService createScheduler() { @@ -414,6 +427,8 @@ public class AlarmGroupReduce implements DisposableBean { private int pendingRuns; + private boolean cancelled; + private ScheduledDispatchTask(ExecutorService executor, Runnable task) { this.executor = executor; this.task = task; @@ -422,6 +437,9 @@ public class AlarmGroupReduce implements DisposableBean { private void dispatch() { boolean shouldSchedule; synchronized (this) { + if (cancelled) { + return; + } pendingRuns++; shouldSchedule = !running; if (shouldSchedule) { @@ -452,6 +470,11 @@ public class AlarmGroupReduce implements DisposableBean { private void scheduleNextIfNeeded() { boolean shouldSchedule; synchronized (this) { + if (cancelled) { + pendingRuns = 0; + running = false; + return; + } pendingRuns = Math.max(0, pendingRuns - 1); shouldSchedule = pendingRuns > 0; if (!shouldSchedule) { @@ -461,5 +484,10 @@ public class AlarmGroupReduce implements DisposableBean { } scheduleRun(); } + + private synchronized void cancel() { + cancelled = true; + pendingRuns = 0; + } } } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java index 2880a16a2f..a300784d1b 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java @@ -78,29 +78,26 @@ public class AlarmInhibitReduce implements DisposableBean { */ private final long sourceAlertTtl; - private final ScheduledExecutorService cleanupScheduler; + private final AlertInhibitDao alertInhibitDao; + private final VirtualThreadProperties virtualThreadProperties; + private ScheduledExecutorService cleanupScheduler; - private final ExecutorService cleanupExecutor; + private ExecutorService cleanupExecutor; - private final ScheduledDispatchTask cleanupTask; + private ScheduledDispatchTask cleanupTask; public AlarmInhibitReduce(AlarmSilenceReduce alarmSilenceReduce, AlertInhibitDao alertInhibitDao , AlerterProperties alerterProperties) { - this(alarmSilenceReduce, alertInhibitDao, alerterProperties, VirtualThreadProperties.defaults(), true); + this(alarmSilenceReduce, alertInhibitDao, alerterProperties, VirtualThreadProperties.defaults()); } @Autowired public AlarmInhibitReduce(AlarmSilenceReduce alarmSilenceReduce, AlertInhibitDao alertInhibitDao, AlerterProperties alerterProperties, VirtualThreadProperties virtualThreadProperties) { - this(alarmSilenceReduce, alertInhibitDao, alerterProperties, virtualThreadProperties, true); - } - - AlarmInhibitReduce(AlarmSilenceReduce alarmSilenceReduce, AlertInhibitDao alertInhibitDao, - AlerterProperties alerterProperties, VirtualThreadProperties virtualThreadProperties, - boolean autoStart) { this.alarmSilenceReduce = alarmSilenceReduce; - VirtualThreadProperties properties = - virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + this.alertInhibitDao = alertInhibitDao; + this.virtualThreadProperties = virtualThreadProperties == null + ? VirtualThreadProperties.defaults() : virtualThreadProperties; if (alerterProperties.getInhibit() != null && alerterProperties.getInhibit().getTtl() > 0) { this.sourceAlertTtl = alerterProperties.getInhibit().getTtl(); } else { @@ -108,34 +105,49 @@ public class AlarmInhibitReduce implements DisposableBean { } inhibitRules = new ConcurrentHashMap<>(8); sourceAlertCache = new ConcurrentHashMap<>(8); - this.cleanupScheduler = createCleanupScheduler(); - this.cleanupExecutor = createCleanupExecutor(properties); - this.cleanupTask = new ScheduledDispatchTask(cleanupExecutor, this::runCleanupCache); - List inhibits = alertInhibitDao.findAlertInhibitsByEnableIsTrue(); - refreshInhibitRules(inhibits); - if (autoStart) { - startScheduledCleanupCache(); - } } - private void startScheduledCleanupCache() { - cleanupScheduler.scheduleAtFixedRate(this::dispatchCleanupCache, CHECK_INTERVAL, CHECK_INTERVAL, + synchronized void start() { + if (cleanupScheduler != null) { + return; + } + cleanupScheduler = createCleanupScheduler(); + cleanupExecutor = createCleanupExecutor(virtualThreadProperties); + ScheduledDispatchTask currentCleanupTask = + new ScheduledDispatchTask(cleanupExecutor, this::runCleanupCache); + cleanupTask = currentCleanupTask; + refreshInhibitRules(alertInhibitDao.findAlertInhibitsByEnableIsTrue()); + startScheduledCleanupCache(currentCleanupTask); + } + + private void startScheduledCleanupCache(ScheduledDispatchTask currentCleanupTask) { + cleanupScheduler.scheduleAtFixedRate(currentCleanupTask::dispatch, CHECK_INTERVAL, CHECK_INTERVAL, TimeUnit.MILLISECONDS); } - void dispatchCleanupCache() { - cleanupTask.dispatch(); + synchronized void dispatchCleanupCache() { + if (cleanupTask != null) { + cleanupTask.dispatch(); + } } void beforeCleanupCacheRun() { } @Override - public void destroy() { - cleanupScheduler.shutdownNow(); + public synchronized void destroy() { + if (cleanupTask != null) { + cleanupTask.cancel(); + } + if (cleanupScheduler != null) { + cleanupScheduler.shutdownNow(); + cleanupScheduler = null; + } if (cleanupExecutor != null) { cleanupExecutor.shutdownNow(); + cleanupExecutor = null; } + cleanupTask = null; } private ScheduledExecutorService createCleanupScheduler() { @@ -389,6 +401,8 @@ public class AlarmInhibitReduce implements DisposableBean { private int pendingRuns; + private boolean cancelled; + private ScheduledDispatchTask(ExecutorService executor, Runnable task) { this.executor = executor; this.task = task; @@ -397,6 +411,9 @@ public class AlarmInhibitReduce implements DisposableBean { private void dispatch() { boolean shouldSchedule; synchronized (this) { + if (cancelled) { + return; + } pendingRuns++; shouldSchedule = !running; if (shouldSchedule) { @@ -427,6 +444,11 @@ public class AlarmInhibitReduce implements DisposableBean { private void scheduleNextIfNeeded() { boolean shouldSchedule; synchronized (this) { + if (cancelled) { + pendingRuns = 0; + running = false; + return; + } pendingRuns = Math.max(0, pendingRuns - 1); shouldSchedule = pendingRuns > 0; if (!shouldSchedule) { @@ -436,5 +458,10 @@ public class AlarmInhibitReduce implements DisposableBean { } scheduleRun(); } + + private synchronized void cancel() { + cancelled = true; + pendingRuns = 0; + } } } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmReduceLifecycle.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmReduceLifecycle.java new file mode 100644 index 0000000000..6e57e34d6b --- /dev/null +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmReduceLifecycle.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.alert.reduce; + +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; +import org.springframework.boot.CommandLineRunner; +import org.springframework.stereotype.Component; + +/** Loads reducer rules and starts cleanup/grouping workers only in normal runtime. */ +@Component +@ConditionalOnNormalBusinessRuntime +public final class AlarmReduceLifecycle implements CommandLineRunner { + + private final AlarmInhibitReduce inhibitReduce; + private final AlarmGroupReduce groupReduce; + + public AlarmReduceLifecycle(AlarmInhibitReduce inhibitReduce, AlarmGroupReduce groupReduce) { + this.inhibitReduce = inhibitReduce; + this.groupReduce = groupReduce; + } + + @Override + public void run(String... args) { + inhibitReduce.start(); + groupReduce.start(); + } +} diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/service/impl/NoticeConfigServiceImpl.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/service/impl/NoticeConfigServiceImpl.java index d28b91e71c..3fcd1f4c30 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/service/impl/NoticeConfigServiceImpl.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/service/impl/NoticeConfigServiceImpl.java @@ -34,7 +34,6 @@ import org.apache.hertzbeat.alert.service.NoticeTemplateMutationException; import org.apache.hertzbeat.alert.service.NoticeConfigService; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.CommandLineRunner; import org.springframework.context.annotation.Lazy; import org.springframework.core.Ordered; import org.springframework.core.annotation.Order; @@ -69,7 +68,7 @@ import java.util.stream.Collectors; @Order(value = Ordered.HIGHEST_PRECEDENCE) @Transactional(rollbackFor = Exception.class) @Slf4j -public class NoticeConfigServiceImpl implements NoticeConfigService, CommandLineRunner { +public class NoticeConfigServiceImpl implements NoticeConfigService { private static final Map PRESET_TEMPLATE = new HashMap<>(16); @@ -372,8 +371,7 @@ public class NoticeConfigServiceImpl implements NoticeConfigService, CommandLine CacheFactory.clearNoticeCache(); } - @Override - public void run(String... args) throws Exception { + void loadPresetTemplates() { try { log.info("load default notice template in internal jar"); PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(); diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/service/impl/NoticeTemplateInitializer.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/service/impl/NoticeTemplateInitializer.java new file mode 100644 index 0000000000..d319cade3e --- /dev/null +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/service/impl/NoticeTemplateInitializer.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.alert.service.impl; + +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; +import org.springframework.boot.CommandLineRunner; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; + +/** Loads preset templates only after the business runtime is opened. */ +@Component +@Order(Ordered.HIGHEST_PRECEDENCE) +@ConditionalOnNormalBusinessRuntime +public final class NoticeTemplateInitializer implements CommandLineRunner { + + private final NoticeConfigServiceImpl noticeConfigService; + + public NoticeTemplateInitializer(NoticeConfigServiceImpl noticeConfigService) { + this.noticeConfigService = noticeConfigService; + } + + @Override + public void run(String... args) { + noticeConfigService.loadPresetTemplates(); + } +} diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java index 9d104ab064..d47399f1a2 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java @@ -20,8 +20,10 @@ package org.apache.hertzbeat.alert.calculate.periodic; import static org.apache.hertzbeat.common.constants.CommonConstants.METRIC_ALERT_THRESHOLD_TYPE_PERIODIC; import static org.apache.hertzbeat.common.constants.CommonConstants.TRACE_ALERT_THRESHOLD_TYPE_PERIODIC; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -64,12 +66,13 @@ class PeriodicAlertRuleSchedulerTest { void setUp() { scheduler = new PeriodicAlertRuleScheduler(metricsCalculator, logCalculator, traceCalculator, alertDefineDao, VirtualThreadProperties.defaults()); + scheduler.start(); } @AfterEach void tearDown() { if (scheduler != null) { - scheduler.destroy(); + scheduler.stop(); } } @@ -151,9 +154,10 @@ class PeriodicAlertRuleSchedulerTest { @Test void updateScheduleHonorsConfiguredGlobalPeriodicConcurrencyLimit() throws InterruptedException { - scheduler.destroy(); + scheduler.stop(); scheduler = new PeriodicAlertRuleScheduler(metricsCalculator, logCalculator, traceCalculator, alertDefineDao, periodicProperties(1)); + scheduler.start(); CountDownLatch firstStarted = new CountDownLatch(1); CountDownLatch releaseFirst = new CountDownLatch(1); @@ -191,7 +195,9 @@ class PeriodicAlertRuleSchedulerTest { } @Test - void runLoadsPeriodicTraceRulesAtStartup() throws Exception { + void startLoadsPeriodicTraceRulesAtStartup() { + scheduler.stop(); + clearInvocations(alertDefineDao); when(alertDefineDao.findAlertDefinesByTypeAndEnableTrue(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC)) .thenReturn(java.util.List.of()); when(alertDefineDao.findAlertDefinesByTypeAndEnableTrue( @@ -200,7 +206,7 @@ class PeriodicAlertRuleSchedulerTest { when(alertDefineDao.findAlertDefinesByTypeAndEnableTrue(TRACE_ALERT_THRESHOLD_TYPE_PERIODIC)) .thenReturn(java.util.List.of(traceRule(6L))); - scheduler.run(); + scheduler.start(); verify(alertDefineDao).findAlertDefinesByTypeAndEnableTrue(TRACE_ALERT_THRESHOLD_TYPE_PERIODIC); } @@ -218,6 +224,40 @@ class PeriodicAlertRuleSchedulerTest { assertTrue(latch.await(5, TimeUnit.SECONDS)); } + @Test + void stopDuringPendingExecutionIsIdempotentAndDoesNotResubmit() throws InterruptedException { + CountDownLatch started = new CountDownLatch(1); + CountDownLatch interrupted = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger invocations = new AtomicInteger(); + doAnswer(invocation -> { + int current = invocations.incrementAndGet(); + if (current == 1) { + started.countDown(); + try { + Thread.sleep(5000L); + } catch (InterruptedException e) { + interrupted.countDown(); + Thread.currentThread().interrupt(); + } + } else { + secondStarted.countDown(); + } + return null; + }).when(metricsCalculator).calculate(any(AlertDefine.class)); + + scheduler.updateSchedule(metricRule(8L)); + assertTrue(started.await(5, TimeUnit.SECONDS)); + Thread.sleep(1200L); + + scheduler.stop(); + scheduler.stop(); + + assertTrue(interrupted.await(5, TimeUnit.SECONDS)); + assertFalse(secondStarted.await(1500, TimeUnit.MILLISECONDS)); + assertEquals(1, invocations.get()); + } + private AlertDefine metricRule(Long id) { return AlertDefine.builder() .id(id) diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java index 939440b95d..9007b9c3f7 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java @@ -44,7 +44,10 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import java.util.Arrays; import java.util.Collections; @@ -69,6 +72,21 @@ import org.mockito.MockitoAnnotations; */ class AlarmGroupReduceTest { + @Test + void constructorIsPassiveAndLifecycleIsIdempotent() { + clearInvocations(alertGroupConvergeDao); + AlarmGroupReduce inactive = new AlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao, + new VirtualThreadProperties()); + + verifyNoInteractions(alertGroupConvergeDao); + + inactive.start(); + inactive.start(); + verify(alertGroupConvergeDao, times(1)).findAlertGroupConvergesByEnableIsTrue(); + inactive.destroy(); + inactive.destroy(); + } + @Mock private AlarmInhibitReduce alarmInhibitReduce; @@ -83,7 +101,8 @@ class AlarmGroupReduceTest { when(alertGroupConvergeDao.findAlertGroupConvergesByEnableIsTrue()) .thenReturn(Collections.emptyList()); alarmGroupReduce = new AlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao, - new VirtualThreadProperties(), false); + new VirtualThreadProperties()); + alarmGroupReduce.start(); } @AfterEach @@ -136,6 +155,7 @@ class AlarmGroupReduceTest { alarmGroupReduce.destroy(); alarmGroupReduce = new TestAlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao, new VirtualThreadProperties(), latch, virtualThread, null, null, null, null, null); + alarmGroupReduce.start(); alarmGroupReduce.dispatchCheckAndSendGroups(); @@ -143,6 +163,13 @@ class AlarmGroupReduceTest { assertTrue(virtualThread.get()); } + @Test + void dispatchAfterDestroyIsSafeNoOp() { + alarmGroupReduce.destroy(); + + alarmGroupReduce.dispatchCheckAndSendGroups(); + } + @Test void dispatchCheckAndSendGroupsDoesNotRunConcurrently() throws Exception { CountDownLatch firstStarted = new CountDownLatch(1); @@ -153,6 +180,7 @@ class AlarmGroupReduceTest { alarmGroupReduce = new TestAlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao, new VirtualThreadProperties(), null, null, firstStarted, releaseFirst, secondStarted, maxConcurrent, new AtomicInteger()); + alarmGroupReduce.start(); alarmGroupReduce.dispatchCheckAndSendGroups(); assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); @@ -165,6 +193,29 @@ class AlarmGroupReduceTest { assertEquals(1, maxConcurrent.get()); } + @Test + void destroyWhileCheckIsRunningDropsPendingDispatch() throws Exception { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger invocations = new AtomicInteger(); + alarmGroupReduce.destroy(); + alarmGroupReduce = new TestAlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao, + new VirtualThreadProperties(), null, null, firstStarted, releaseFirst, secondStarted, + new AtomicInteger(), invocations); + alarmGroupReduce.start(); + + alarmGroupReduce.dispatchCheckAndSendGroups(); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + alarmGroupReduce.dispatchCheckAndSendGroups(); + + alarmGroupReduce.destroy(); + alarmGroupReduce.dispatchCheckAndSendGroups(); + + assertFalse(secondStarted.await(500, TimeUnit.MILLISECONDS)); + assertEquals(1, invocations.get()); + } + private Map createLabels(String... keyValues) { Map labels = new HashMap<>(); for (int i = 0; i < keyValues.length; i += 2) { @@ -196,7 +247,7 @@ class AlarmGroupReduceTest { AtomicBoolean virtualThread, CountDownLatch firstStarted, CountDownLatch releaseFirst, CountDownLatch secondStarted, AtomicInteger maxConcurrent, AtomicInteger invocations) { - super(alarmInhibitReduce, alertGroupConvergeDao, properties, false); + super(alarmInhibitReduce, alertGroupConvergeDao, properties); this.virtualThreadLatch = virtualThreadLatch; this.virtualThread = virtualThread; this.firstStarted = firstStarted; diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduceTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduceTest.java index 6991dace48..9324c5fdcd 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduceTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduceTest.java @@ -41,8 +41,11 @@ package org.apache.hertzbeat.alert.reduce; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import java.util.Arrays; import java.util.Collections; @@ -73,6 +76,21 @@ import org.mockito.MockitoAnnotations; */ class AlarmInhibitReduceTest { + @Test + void constructorIsPassiveAndLifecycleIsIdempotent() { + clearInvocations(alertInhibitDao); + AlarmInhibitReduce inactive = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties, + new VirtualThreadProperties()); + + verifyNoInteractions(alertInhibitDao); + + inactive.start(); + inactive.start(); + verify(alertInhibitDao, times(1)).findAlertInhibitsByEnableIsTrue(); + inactive.destroy(); + inactive.destroy(); + } + @Mock private AlertInhibitDao alertInhibitDao; @@ -96,7 +114,8 @@ class AlarmInhibitReduceTest { when(alerterProperties.getInhibit()).thenReturn(inhibitProperties); alarmInhibitReduce = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties, - new VirtualThreadProperties(), false); + new VirtualThreadProperties()); + alarmInhibitReduce.start(); } @AfterEach @@ -306,7 +325,8 @@ class AlarmInhibitReduceTest { when(alerterProperties.getInhibit()).thenReturn(inhibitProperties); alarmInhibitReduce.destroy(); alarmInhibitReduce = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties, - new VirtualThreadProperties(), false); + new VirtualThreadProperties()); + alarmInhibitReduce.start(); AlertInhibit rule = AlertInhibit.builder() .id(1L) @@ -347,6 +367,7 @@ class AlarmInhibitReduceTest { alarmInhibitReduce.destroy(); alarmInhibitReduce = new TestAlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties, new VirtualThreadProperties(), latch, virtualThread, null, null, null, null, null); + alarmInhibitReduce.start(); alarmInhibitReduce.dispatchCleanupCache(); @@ -354,6 +375,13 @@ class AlarmInhibitReduceTest { assertTrue(virtualThread.get()); } + @Test + void dispatchAfterDestroyIsSafeNoOp() { + alarmInhibitReduce.destroy(); + + alarmInhibitReduce.dispatchCleanupCache(); + } + @Test void dispatchCleanupCacheDoesNotRunConcurrently() throws Exception { CountDownLatch firstStarted = new CountDownLatch(1); @@ -364,6 +392,7 @@ class AlarmInhibitReduceTest { alarmInhibitReduce = new TestAlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties, new VirtualThreadProperties(), null, null, firstStarted, releaseFirst, secondStarted, maxConcurrent, new AtomicInteger()); + alarmInhibitReduce.start(); alarmInhibitReduce.dispatchCleanupCache(); assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); @@ -376,6 +405,29 @@ class AlarmInhibitReduceTest { assertEquals(1, maxConcurrent.get()); } + @Test + void destroyWhileCleanupIsRunningDropsPendingDispatch() throws Exception { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger invocations = new AtomicInteger(); + alarmInhibitReduce.destroy(); + alarmInhibitReduce = new TestAlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties, + new VirtualThreadProperties(), null, null, firstStarted, releaseFirst, secondStarted, + new AtomicInteger(), invocations); + alarmInhibitReduce.start(); + + alarmInhibitReduce.dispatchCleanupCache(); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + alarmInhibitReduce.dispatchCleanupCache(); + + alarmInhibitReduce.destroy(); + alarmInhibitReduce.dispatchCleanupCache(); + + assertFalse(secondStarted.await(500, TimeUnit.MILLISECONDS)); + assertEquals(1, invocations.get()); + } + private GroupAlert createGroupAlert(String status, Map labels, List alerts) { return GroupAlert.builder() .status(status) @@ -424,7 +476,7 @@ class AlarmInhibitReduceTest { CountDownLatch firstStarted, CountDownLatch releaseFirst, CountDownLatch secondStarted, AtomicInteger maxConcurrent, AtomicInteger invocations) { - super(alarmSilenceReduce, alertInhibitDao, alerterProperties, properties, false); + super(alarmSilenceReduce, alertInhibitDao, alerterProperties, properties); this.virtualThreadLatch = virtualThreadLatch; this.virtualThread = virtualThread; this.firstStarted = firstStarted; diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/runtime/BusinessRuntimeGate.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/runtime/BusinessRuntimeGate.java new file mode 100644 index 0000000000..63b4343e24 --- /dev/null +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/runtime/BusinessRuntimeGate.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.runtime; + +import java.util.Objects; + +/** Represents the business-runtime boundary decided before an application context starts. */ +public final class BusinessRuntimeGate { + + private final RuntimeMode mode; + + private BusinessRuntimeGate(RuntimeMode mode) { + this.mode = Objects.requireNonNull(mode, "mode"); + } + + public static BusinessRuntimeGate fixed(RuntimeMode mode) { + return new BusinessRuntimeGate(mode); + } + + public RuntimeMode mode() { + return mode; + } + + public boolean isOpen() { + return mode == RuntimeMode.NORMAL; + } +} diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/runtime/RuntimeMode.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/runtime/RuntimeMode.java new file mode 100644 index 0000000000..07a28dd7ef --- /dev/null +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/runtime/RuntimeMode.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.runtime; + +import java.util.Locale; + +/** Application runtime scope selected before a Spring context starts. */ +public enum RuntimeMode { + SETUP_ONLY("setup_only"), + FULL_SETUP_GATED("full_setup_gated"), + NORMAL("normal"), + RECOVERY("recovery"); + + public static final String PROPERTY_NAME = "hertzbeat.runtime.mode"; + + private final String value; + + RuntimeMode(String value) { + this.value = value; + } + + public String value() { + return value; + } + + public static RuntimeMode fromProperty(String value) { + if (value == null) { + return NORMAL; + } + String normalized = value.trim().toLowerCase(Locale.ROOT); + for (RuntimeMode mode : values()) { + if (mode.value.equals(normalized)) { + return mode; + } + } + throw new IllegalArgumentException("Unsupported runtime mode"); + } +} diff --git a/hertzbeat-common-core/src/test/java/org/apache/hertzbeat/common/runtime/BusinessRuntimeGateTest.java b/hertzbeat-common-core/src/test/java/org/apache/hertzbeat/common/runtime/BusinessRuntimeGateTest.java new file mode 100644 index 0000000000..d2c47c7e7b --- /dev/null +++ b/hertzbeat-common-core/src/test/java/org/apache/hertzbeat/common/runtime/BusinessRuntimeGateTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.runtime; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class BusinessRuntimeGateTest { + + @Test + void opensOnlyForNormalRuntime() { + for (RuntimeMode mode : RuntimeMode.values()) { + BusinessRuntimeGate gate = BusinessRuntimeGate.fixed(mode); + assertEquals(mode, gate.mode()); + assertEquals(mode == RuntimeMode.NORMAL, gate.isOpen()); + } + } + + @Test + void missingModePreservesExistingNormalStartup() { + assertEquals(RuntimeMode.NORMAL, RuntimeMode.fromProperty(null)); + } +} diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/runtime/BusinessRuntimeConfiguration.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/runtime/BusinessRuntimeConfiguration.java new file mode 100644 index 0000000000..c291e37fa2 --- /dev/null +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/runtime/BusinessRuntimeConfiguration.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.runtime; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; + +/** Provides the immutable runtime gate selected before this context was launched. */ +@Configuration(proxyBeanMethods = false) +public class BusinessRuntimeConfiguration { + + @Bean + @ConditionalOnMissingBean + public BusinessRuntimeGate businessRuntimeGate(Environment environment) { + return BusinessRuntimeGate.fixed(RuntimeMode.fromProperty(environment.getProperty(RuntimeMode.PROPERTY_NAME))); + } +} diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/runtime/ConditionalOnNormalBusinessRuntime.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/runtime/ConditionalOnNormalBusinessRuntime.java new file mode 100644 index 0000000000..15d2c43869 --- /dev/null +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/runtime/ConditionalOnNormalBusinessRuntime.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.runtime; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; + +/** Registers an active side-effect boundary only in the normal full runtime. */ +@Target({ElementType.TYPE, ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnProperty(name = RuntimeMode.PROPERTY_NAME, havingValue = "normal", matchIfMissing = true) +public @interface ConditionalOnNormalBusinessRuntime { +} diff --git a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/runtime/NormalBusinessRuntimeConditionTest.java b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/runtime/NormalBusinessRuntimeConditionTest.java new file mode 100644 index 0000000000..ccff8fb9ee --- /dev/null +++ b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/runtime/NormalBusinessRuntimeConditionTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.runtime; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +class NormalBusinessRuntimeConditionTest { + + private final ApplicationContextRunner runner = new ApplicationContextRunner() + .withUserConfiguration(ConditionalConfiguration.class); + + @Test + void fullSetupGatedContextStartsWithoutBusinessSideEffectBean() { + runner.withPropertyValues(RuntimeMode.PROPERTY_NAME + "=full_setup_gated").run(context -> { + assertTrue(context.isRunning()); + assertFalse(context.containsBean("businessSideEffect")); + }); + } + + @Test + void normalContextRegistersBusinessSideEffectBean() { + runner.withPropertyValues(RuntimeMode.PROPERTY_NAME + "=normal").run(context -> + assertTrue(context.containsBean("businessSideEffect"))); + } + + @Test + void missingRuntimeModePreservesExistingNormalStartup() { + runner.run(context -> assertTrue(context.containsBean("businessSideEffect"))); + } + + @Configuration(proxyBeanMethods = false) + static class ConditionalConfiguration { + + @Bean + @ConditionalOnNormalBusinessRuntime + String businessSideEffect() { + return "started"; + } + } +} diff --git a/hertzbeat-grafana/src/main/java/org/apache/hertzbeat/grafana/config/GrafanaInit.java b/hertzbeat-grafana/src/main/java/org/apache/hertzbeat/grafana/config/GrafanaInit.java index 8e85a6725e..9bffa88d51 100644 --- a/hertzbeat-grafana/src/main/java/org/apache/hertzbeat/grafana/config/GrafanaInit.java +++ b/hertzbeat-grafana/src/main/java/org/apache/hertzbeat/grafana/config/GrafanaInit.java @@ -18,6 +18,7 @@ package org.apache.hertzbeat.grafana.config; import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.apache.hertzbeat.grafana.service.DatasourceService; import org.apache.hertzbeat.grafana.service.ServiceAccountService; import org.springframework.beans.factory.annotation.Autowired; @@ -28,6 +29,7 @@ import org.springframework.stereotype.Component; * grafana init */ @Component +@ConditionalOnNormalBusinessRuntime @Slf4j public class GrafanaInit implements CommandLineRunner { @Autowired diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseManager.java b/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseManager.java index 31a537c6f2..5172c25495 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseManager.java +++ b/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseManager.java @@ -56,24 +56,35 @@ public class LogSseManager { private final Map emitters = new ConcurrentHashMap<>(); private final Queue logQueue = new ConcurrentLinkedQueue<>(); - private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(r -> { - Thread t = new Thread(r, "sse-batch-scheduler"); - t.setDaemon(true); - return t; - }); - private final ExecutorService senderPool = Executors.newCachedThreadPool(r -> { - Thread t = new Thread(r, "sse-sender"); - t.setDaemon(true); - return t; - }); - private final AtomicLong queueSize = new AtomicLong(0); + private ScheduledExecutorService scheduler; + private ExecutorService senderPool; - public LogSseManager() { - scheduler.scheduleAtFixedRate(this::flushBatch, BATCH_INTERVAL_MS, BATCH_INTERVAL_MS, TimeUnit.MILLISECONDS); + synchronized void start() { + if (scheduler != null) { + return; + } + scheduler = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "sse-batch-scheduler"); + t.setDaemon(true); + return t; + }); + senderPool = Executors.newCachedThreadPool(r -> { + Thread t = new Thread(r, "sse-sender"); + t.setDaemon(true); + return t; + }); + ExecutorService currentSenderPool = senderPool; + scheduler.scheduleAtFixedRate( + () -> flushBatch(currentSenderPool), BATCH_INTERVAL_MS, BATCH_INTERVAL_MS, TimeUnit.MILLISECONDS); } + private final AtomicLong queueSize = new AtomicLong(0); + @PreDestroy - public void shutdown() { + public synchronized void shutdown() { + if (scheduler == null) { + return; + } scheduler.shutdown(); senderPool.shutdown(); try { @@ -84,6 +95,8 @@ public class LogSseManager { } scheduler.shutdownNow(); senderPool.shutdownNow(); + scheduler = null; + senderPool = null; } /** @@ -117,7 +130,13 @@ public class LogSseManager { /** * Flush queued logs to all subscribers in batch */ - private void flushBatch() { + synchronized void flushBatch() { + if (senderPool != null) { + flushBatch(senderPool); + } + } + + private void flushBatch(ExecutorService currentSenderPool) { try { if (logQueue.isEmpty() || emitters.isEmpty()) { return; @@ -140,7 +159,7 @@ public class LogSseManager { SseSubscriber subscriber = e.getValue(); List filtered = filterLogs(batch, subscriber.filters); if (!filtered.isEmpty()) { - senderPool.submit(() -> sendToSubscriber(clientId, subscriber.emitter, filtered)); + currentSenderPool.submit(() -> sendToSubscriber(clientId, subscriber.emitter, filtered)); } } } catch (Exception e) { diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseManagerLifecycle.java b/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseManagerLifecycle.java new file mode 100644 index 0000000000..fe413bf8fd --- /dev/null +++ b/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseManagerLifecycle.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.log.notice; + +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; +import org.springframework.boot.CommandLineRunner; +import org.springframework.stereotype.Component; + +/** Starts log SSE delivery threads only in normal runtime. */ +@Component("legacyLogSseManagerLifecycle") +@ConditionalOnNormalBusinessRuntime +public final class LogSseManagerLifecycle implements CommandLineRunner { + + private final LogSseManager manager; + + public LogSseManagerLifecycle(LogSseManager manager) { + this.manager = manager; + } + + @Override + public void run(String... args) { + manager.start(); + } +} diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/notice/LogSseManagerTest.java b/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/notice/LogSseManagerTest.java index 3f2c39ebeb..a292a4b3eb 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/notice/LogSseManagerTest.java +++ b/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/notice/LogSseManagerTest.java @@ -51,6 +51,7 @@ class LogSseManagerTest { @BeforeEach void setUp() { logSseManager = new LogSseManager(); + logSseManager.start(); } @AfterEach @@ -172,6 +173,17 @@ class LogSseManagerTest { }); } + @Test + void flushAfterShutdownIsSafeNoOp() { + logSseManager.shutdown(); + logSseManager.broadcast(createLogEntry("INFO", "after shutdown")); + + logSseManager.flushBatch(); + logSseManager.shutdown(); + + assertEquals(1, logSseManager.getQueueSize()); + } + /** * Helper method to create a subscriber and inject a mock emitter for testing */ @@ -190,4 +202,4 @@ class LogSseManagerTest { .body(body) .build(); } -} \ No newline at end of file +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryWorker.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryWorker.java index a80d4041f3..e214124313 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryWorker.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryWorker.java @@ -28,6 +28,7 @@ import org.apache.hertzbeat.common.entity.manager.Param; import org.apache.hertzbeat.common.entity.message.CollectRep; import org.apache.hertzbeat.common.queue.CommonDataQueue; import org.apache.hertzbeat.common.support.exception.CommonDataQueueUnknownException; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.apache.hertzbeat.common.util.BackoffUtils; import org.apache.hertzbeat.common.util.ExponentialBackoff; import org.apache.hertzbeat.manager.dao.CollectorMonitorBindDao; @@ -52,6 +53,7 @@ import java.util.stream.Collectors; */ @Slf4j @Component +@ConditionalOnNormalBusinessRuntime public class ServiceDiscoveryWorker implements InitializingBean { private static final String FILED_HOST = "host"; diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatus.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatus.java index 05e470ed6e..4831c9ee23 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatus.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatus.java @@ -70,61 +70,81 @@ public class CalculateStatus implements DisposableBean { private final MonitorDao monitorDao; private final int intervals; + private final VirtualThreadProperties virtualThreadProperties; - private final ScheduledExecutorService calculateScheduler; + private ScheduledExecutorService calculateScheduler; - private final ScheduledExecutorService combineHistoryScheduler; + private ScheduledExecutorService combineHistoryScheduler; - private final ExecutorService calculateExecutor; + private ExecutorService calculateExecutor; - private final ExecutorService combineHistoryExecutor; + private ExecutorService combineHistoryExecutor; - private final ScheduledDispatchTask calculateTask; + private ScheduledDispatchTask calculateTask; - private final ScheduledDispatchTask combineHistoryTask; + private ScheduledDispatchTask combineHistoryTask; + + private boolean started; public CalculateStatus(StatusPageOrgDao statusPageOrgDao, StatusPageComponentDao statusPageComponentDao, StatusProperties statusProperties, StatusPageHistoryDao statusPageHistoryDao, MonitorDao monitorDao) { this(statusPageOrgDao, statusPageComponentDao, statusProperties, statusPageHistoryDao, monitorDao, - VirtualThreadProperties.defaults(), true); + VirtualThreadProperties.defaults()); } @Autowired public CalculateStatus(StatusPageOrgDao statusPageOrgDao, StatusPageComponentDao statusPageComponentDao, StatusProperties statusProperties, StatusPageHistoryDao statusPageHistoryDao, MonitorDao monitorDao, VirtualThreadProperties virtualThreadProperties) { - this(statusPageOrgDao, statusPageComponentDao, statusProperties, statusPageHistoryDao, monitorDao, - virtualThreadProperties, true); - } - - CalculateStatus(StatusPageOrgDao statusPageOrgDao, StatusPageComponentDao statusPageComponentDao, - StatusProperties statusProperties, StatusPageHistoryDao statusPageHistoryDao, - MonitorDao monitorDao, VirtualThreadProperties virtualThreadProperties, boolean autoStart) { this.statusPageOrgDao = statusPageOrgDao; this.monitorDao = monitorDao; this.statusPageComponentDao = statusPageComponentDao; this.statusPageHistoryDao = statusPageHistoryDao; - intervals = statusProperties.getCalculate() == null ? DEFAULT_CALCULATE_INTERVAL_TIME : statusProperties.getCalculate().getInterval(); - this.calculateScheduler = createScheduler("status-page-calculate-%d", "Status calculate has uncaughtException."); - this.combineHistoryScheduler = createScheduler("status-page-history-%d", "History combine has uncaughtException."); - this.calculateExecutor = createVirtualExecutor(virtualThreadProperties, "status-page-calculate-vt-", - "Status calculate worker has uncaughtException."); - this.combineHistoryExecutor = createVirtualExecutor(virtualThreadProperties, "status-page-history-vt-", - "History combine worker has uncaughtException."); - this.calculateTask = new ScheduledDispatchTask(calculateExecutor, this::runCalculate); - this.combineHistoryTask = new ScheduledDispatchTask(combineHistoryExecutor, this::runCombineHistory); - if (autoStart) { - startCalculate(); - startCombineHistory(); + StatusProperties.CalculateProperties calculateProperties = statusProperties.getCalculate(); + intervals = calculateProperties == null + ? DEFAULT_CALCULATE_INTERVAL_TIME : calculateProperties.getInterval(); + this.virtualThreadProperties = virtualThreadProperties == null + ? VirtualThreadProperties.defaults() : virtualThreadProperties; + } + + synchronized void start() { + if (started) { + return; + } + try { + calculateScheduler = createScheduler( + "status-page-calculate-%d", "Status calculate has uncaughtException."); + combineHistoryScheduler = createScheduler( + "status-page-history-%d", "History combine has uncaughtException."); + calculateExecutor = createVirtualExecutor(virtualThreadProperties, "status-page-calculate-vt-", + "Status calculate worker has uncaughtException."); + combineHistoryExecutor = createVirtualExecutor(virtualThreadProperties, "status-page-history-vt-", + "History combine worker has uncaughtException."); + ScheduledDispatchTask currentCalculateTask = + new ScheduledDispatchTask(calculateExecutor, this::runCalculate); + ScheduledDispatchTask currentCombineHistoryTask = + new ScheduledDispatchTask(combineHistoryExecutor, this::runCombineHistory); + calculateTask = currentCalculateTask; + combineHistoryTask = currentCombineHistoryTask; + startCalculate(currentCalculateTask); + startCombineHistory(currentCombineHistoryTask); + started = true; + } catch (RuntimeException | Error e) { + destroy(); + throw e; } } - private void startCalculate() { - calculateScheduler.scheduleAtFixedRate(this::dispatchCalculate, 5, intervals, TimeUnit.SECONDS); + synchronized boolean isStarted() { + return started; } - private void startCombineHistory() { + private void startCalculate(ScheduledDispatchTask currentCalculateTask) { + calculateScheduler.scheduleAtFixedRate(currentCalculateTask::dispatch, 5, intervals, TimeUnit.SECONDS); + } + + private void startCombineHistory(ScheduledDispatchTask currentCombineHistoryTask) { // combine history every day at 1:00 AM LocalDateTime now = LocalDateTime.now(); LocalDateTime nextRun = now.withHour(1).withMinute(0).withSecond(0); @@ -132,7 +152,7 @@ public class CalculateStatus implements DisposableBean { nextRun = nextRun.plusDays(1); } long delay = Duration.between(now, nextRun).toMillis(); - combineHistoryScheduler.scheduleAtFixedRate(this::dispatchCombineHistory, delay, + combineHistoryScheduler.scheduleAtFixedRate(currentCombineHistoryTask::dispatch, delay, TimeUnit.DAYS.toMillis(1), TimeUnit.MILLISECONDS); } @@ -144,24 +164,45 @@ public class CalculateStatus implements DisposableBean { return intervals; } - void dispatchCalculate() { - calculateTask.dispatch(); + synchronized void dispatchCalculate() { + if (calculateTask != null) { + calculateTask.dispatch(); + } } - void dispatchCombineHistory() { - combineHistoryTask.dispatch(); + synchronized void dispatchCombineHistory() { + if (combineHistoryTask != null) { + combineHistoryTask.dispatch(); + } } @Override - public void destroy() { - calculateScheduler.shutdownNow(); - combineHistoryScheduler.shutdownNow(); + public synchronized void destroy() { + started = false; + if (calculateTask != null) { + calculateTask.cancel(); + } + if (combineHistoryTask != null) { + combineHistoryTask.cancel(); + } + if (calculateScheduler != null) { + calculateScheduler.shutdownNow(); + calculateScheduler = null; + } + if (combineHistoryScheduler != null) { + combineHistoryScheduler.shutdownNow(); + combineHistoryScheduler = null; + } if (calculateExecutor != null) { calculateExecutor.shutdownNow(); + calculateExecutor = null; } if (combineHistoryExecutor != null) { combineHistoryExecutor.shutdownNow(); + combineHistoryExecutor = null; } + calculateTask = null; + combineHistoryTask = null; } private void runCalculate() { @@ -327,6 +368,7 @@ public class CalculateStatus implements DisposableBean { private final Object lock = new Object(); private boolean running; private int pendingRuns; + private boolean cancelled; private ScheduledDispatchTask(ExecutorService executorService, Runnable task) { this.executorService = executorService; @@ -335,10 +377,18 @@ public class CalculateStatus implements DisposableBean { private void dispatch() { if (executorService == null) { + synchronized (lock) { + if (cancelled) { + return; + } + } task.run(); return; } synchronized (lock) { + if (cancelled) { + return; + } if (running) { pendingRuns++; return; @@ -372,7 +422,11 @@ public class CalculateStatus implements DisposableBean { private void onComplete() { boolean shouldRunAgain; synchronized (lock) { - if (pendingRuns > 0) { + if (cancelled) { + running = false; + pendingRuns = 0; + shouldRunAgain = false; + } else if (pendingRuns > 0) { pendingRuns--; shouldRunAgain = true; } else { @@ -384,5 +438,12 @@ public class CalculateStatus implements DisposableBean { submit(); } } + + private void cancel() { + synchronized (lock) { + cancelled = true; + pendingRuns = 0; + } + } } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatusLifecycle.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatusLifecycle.java new file mode 100644 index 0000000000..ea02850232 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatusLifecycle.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.component.status; + +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; +import org.springframework.boot.CommandLineRunner; +import org.springframework.stereotype.Component; + +/** Starts status-page calculation and history consolidation only in normal runtime. */ +@Component +@ConditionalOnNormalBusinessRuntime +public final class CalculateStatusLifecycle implements CommandLineRunner { + + private final CalculateStatus calculateStatus; + + public CalculateStatusLifecycle(CalculateStatus calculateStatus) { + this.calculateStatus = calculateStatus; + } + + @Override + public void run(String... args) { + calculateStatus.start(); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/config/ConfigInitializer.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/config/ConfigInitializer.java index 3886ee41c2..74d21416c7 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/config/ConfigInitializer.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/config/ConfigInitializer.java @@ -22,6 +22,7 @@ import jakarta.annotation.Resource; import java.security.SecureRandom; import lombok.SneakyThrows; import org.apache.commons.lang3.StringUtils; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.apache.hertzbeat.common.util.AesUtil; import org.apache.hertzbeat.manager.pojo.dto.MuteConfig; import org.apache.hertzbeat.manager.pojo.dto.SystemSecret; @@ -42,6 +43,7 @@ import org.springframework.stereotype.Component; */ @Component @Order(value = Ordered.HIGHEST_PRECEDENCE + 2) +@ConditionalOnNormalBusinessRuntime public class ConfigInitializer implements SmartLifecycle { private boolean running = false; diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/SchedulerInit.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/SchedulerInit.java index 83e33e540e..0a094fefb1 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/SchedulerInit.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/SchedulerInit.java @@ -30,6 +30,7 @@ import org.apache.hertzbeat.common.entity.manager.Collector; import org.apache.hertzbeat.common.entity.manager.CollectorMonitorBind; import org.apache.hertzbeat.common.entity.manager.Monitor; import org.apache.hertzbeat.common.entity.manager.Param; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.apache.hertzbeat.manager.config.PrometheusProxyConfig; import org.apache.hertzbeat.manager.dao.CollectorDao; import org.apache.hertzbeat.manager.dao.CollectorMonitorBindDao; @@ -48,6 +49,7 @@ import org.springframework.util.StringUtils; * scheduler init */ @Configuration +@ConditionalOnNormalBusinessRuntime @Order(value = Ordered.LOWEST_PRECEDENCE - 1) @Slf4j public class SchedulerInit implements CommandLineRunner { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServer.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServer.java index 828084cc14..0649e5d5ad 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServer.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServer.java @@ -45,21 +45,14 @@ import org.apache.hertzbeat.remoting.event.NettyEventListener; import org.apache.hertzbeat.remoting.netty.NettyRemotingServer; import org.apache.hertzbeat.remoting.netty.NettyServerConfig; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.CommandLineRunner; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.core.Ordered; -import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; /** * manage server */ @Component -@Order(value = Ordered.LOWEST_PRECEDENCE) -@ConditionalOnProperty(prefix = "scheduler.server", - name = "enabled", havingValue = "true") @Slf4j -public class ManageServer implements CommandLineRunner { +public class ManageServer { private final CollectorJobScheduler collectorJobScheduler; @@ -73,15 +66,18 @@ public class ManageServer implements CommandLineRunner { private ScheduledExecutorService channelSchedule; - private final ExecutorService channelCheckExecutor; + private ChannelCheckGeneration channelCheckGeneration; - private final Object channelCheckLock = new Object(); + private final SchedulerProperties schedulerProperties; - private boolean channelCheckRunning; + private final BackgroundTaskExecutor threadPool; - private boolean channelCheckPending; + private final VirtualThreadProperties virtualThreadProperties; - private RemotingServer remotingServer; + private boolean channelChecksStopped; + + // Lifecycle writes must be visible to command threads; an in-flight command may finish on its captured server. + private volatile RemotingServer remotingServer; private final Map clientChannelTable = new ConcurrentHashMap<>(16); @@ -130,8 +126,10 @@ public class ManageServer implements CommandLineRunner { this.commonDataQueue = commonDataQueue; this.runtimeStatusRegistry = runtimeStatusRegistry; this.runtimeConfigService = runtimeConfigService; - this.channelCheckExecutor = createChannelCheckExecutor(virtualThreadProperties); - this.init(schedulerProperties, threadPool); + this.schedulerProperties = schedulerProperties; + this.threadPool = threadPool; + this.virtualThreadProperties = virtualThreadProperties == null + ? VirtualThreadProperties.defaults() : virtualThreadProperties; } private void init(final SchedulerProperties schedulerProperties, final BackgroundTaskExecutor threadPool) { @@ -154,20 +152,42 @@ public class ManageServer implements CommandLineRunner { this.channelSchedule = Executors.newSingleThreadScheduledExecutor(); } - public void start() { - this.remotingServer.start(); - - this.channelSchedule.scheduleAtFixedRate(this::dispatchChannelHealthCheck, 10, 3, TimeUnit.SECONDS); + public synchronized void start() { + if (remotingServer != null) { + return; + } + try { + init(schedulerProperties, threadPool); + channelChecksStopped = false; + this.remotingServer.start(); + this.channelSchedule.scheduleAtFixedRate(this::dispatchChannelHealthCheck, 10, 3, TimeUnit.SECONDS); + } catch (RuntimeException | Error e) { + try { + shutdown(); + } catch (RuntimeException | Error cleanupFailure) { + e.addSuppressed(cleanupFailure); + } + throw e; + } } - public void shutdown() { - this.remotingServer.shutdown(); - - if (this.channelSchedule != null) { - this.channelSchedule.shutdownNow(); - } - if (this.channelCheckExecutor != null) { - this.channelCheckExecutor.shutdownNow(); + public synchronized void shutdown() { + RemotingServer currentServer = this.remotingServer; + this.remotingServer = null; + try { + if (currentServer != null) { + currentServer.shutdown(); + } + } finally { + if (this.channelSchedule != null) { + this.channelSchedule.shutdownNow(); + this.channelSchedule = null; + } + channelChecksStopped = true; + if (this.channelCheckGeneration != null) { + this.channelCheckGeneration.stop(); + this.channelCheckGeneration = null; + } } } @@ -203,12 +223,16 @@ public class ManageServer implements CommandLineRunner { } public void closeChannel(final String identity) { + RemotingServer currentServer = this.remotingServer; + if (currentServer == null) { + return; + } this.runtimeStatusRegistry.remove(identity); Channel channel = this.getChannel(identity); if (channel != null) { this.collectorJobScheduler.collectorGoOffline(identity); ClusterMsg.Message message = ClusterMsg.Message.newBuilder().setType(ClusterMsg.MessageType.GO_CLOSE).build(); - this.remotingServer.sendMsg(channel, message); + currentServer.sendMsg(channel, message); this.clientChannelTable.remove(identity); log.info("close collect client success, identity: {}", identity); } @@ -220,40 +244,43 @@ public class ManageServer implements CommandLineRunner { } public boolean sendMsg(final String identityId, final ClusterMsg.Message message) { + RemotingServer currentServer = this.remotingServer; + if (currentServer == null) { + return false; + } Channel channel = this.getChannel(identityId); if (channel != null) { - this.remotingServer.sendMsg(channel, message); + currentServer.sendMsg(channel, message); return true; } return false; } public ClusterMsg.Message sendMsgSync(final String identityId, final ClusterMsg.Message message) { + RemotingServer currentServer = this.remotingServer; + if (currentServer == null) { + return null; + } Channel channel = this.getChannel(identityId); if (channel != null) { - return this.remotingServer.sendMsgSync(channel, message, 3000); + return currentServer.sendMsgSync(channel, message, 3000); } return null; } - void dispatchChannelHealthCheck() { - if (channelCheckExecutor == null) { + synchronized void dispatchChannelHealthCheck() { + if (channelChecksStopped) { + return; + } + if (!virtualThreadProperties.enabled()) { runChannelHealthCheck(); return; } - synchronized (channelCheckLock) { - if (channelCheckRunning) { - channelCheckPending = true; - return; - } - channelCheckRunning = true; + if (channelCheckGeneration == null) { + ExecutorService executor = createChannelCheckExecutor(virtualThreadProperties); + channelCheckGeneration = new ChannelCheckGeneration(executor); } - submitChannelHealthCheck(); - } - - @Override - public void run(String... args) throws Exception { - this.start(); + channelCheckGeneration.dispatch(); } /** @@ -292,40 +319,57 @@ public class ManageServer implements CommandLineRunner { .factory()); } - private void submitChannelHealthCheck() { - boolean submitted = false; - try { - channelCheckExecutor.execute(() -> { + private final class ChannelCheckGeneration { + + private final ExecutorService executor; + private boolean running; + private boolean pending; + private boolean stopped; + + private ChannelCheckGeneration(ExecutorService executor) { + this.executor = executor; + } + + private synchronized void dispatch() { + if (stopped) { + return; + } + if (running) { + pending = true; + return; + } + running = true; + submitLocked(); + } + + private void submitLocked() { + executor.execute(() -> { try { runChannelHealthCheck(); } finally { - onChannelHealthCheckComplete(); + onComplete(); } }); - submitted = true; - } finally { - if (!submitted) { - synchronized (channelCheckLock) { - channelCheckRunning = false; - channelCheckPending = false; - } - } } - } - private void onChannelHealthCheckComplete() { - boolean shouldRunAgain; - synchronized (channelCheckLock) { - if (channelCheckPending) { - channelCheckPending = false; - shouldRunAgain = true; - } else { - channelCheckRunning = false; - shouldRunAgain = false; + private synchronized void onComplete() { + if (stopped) { + running = false; + pending = false; + return; } + if (pending) { + pending = false; + submitLocked(); + return; + } + running = false; } - if (shouldRunAgain) { - submitChannelHealthCheck(); + + private synchronized void stop() { + stopped = true; + pending = false; + executor.shutdownNow(); } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServerLifecycle.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServerLifecycle.java new file mode 100644 index 0000000000..d702856795 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServerLifecycle.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.scheduler.netty; + +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.boot.CommandLineRunner; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; + +/** Owns the Collector command server socket and health-check executors only in normal runtime. */ +@Component +@Order(value = Ordered.LOWEST_PRECEDENCE) +@ConditionalOnNormalBusinessRuntime +@ConditionalOnProperty(prefix = "scheduler.server", name = "enabled", havingValue = "true") +public final class ManageServerLifecycle implements CommandLineRunner, DisposableBean { + + private final ManageServer manageServer; + + public ManageServerLifecycle(ManageServer manageServer) { + this.manageServer = manageServer; + } + + @Override + public void run(String... args) { + manageServer.start(); + } + + @Override + public void destroy() { + manageServer.shutdown(); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/entity/LocalTopologyDemoRelationSeeder.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/entity/LocalTopologyDemoRelationSeeder.java index 04fe9f0c9c..3f4064059e 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/entity/LocalTopologyDemoRelationSeeder.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/entity/LocalTopologyDemoRelationSeeder.java @@ -26,6 +26,7 @@ import org.apache.hertzbeat.common.entity.manager.EntityIdentity; import org.apache.hertzbeat.common.entity.manager.EntityRelation; import org.apache.hertzbeat.common.entity.manager.ObserveEntity; import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.apache.hertzbeat.manager.dao.EntityIdentityDao; import org.apache.hertzbeat.manager.dao.EntityRelationDao; import org.apache.hertzbeat.manager.dao.ObserveEntityDao; @@ -39,6 +40,7 @@ import org.springframework.transaction.annotation.Transactional; * Seeds local-only demo topology relations so fresh local H2 catalogs do not render node-only graphs. */ @Component +@ConditionalOnNormalBusinessRuntime @Profile("local") @Slf4j public class LocalTopologyDemoRelationSeeder implements ApplicationRunner { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/AppServiceImpl.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/AppServiceImpl.java index 159faa7c81..43f529cc8f 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/AppServiceImpl.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/AppServiceImpl.java @@ -54,7 +54,6 @@ import org.apache.hertzbeat.manager.service.AppService; import org.apache.hertzbeat.manager.service.MonitorService; import org.apache.hertzbeat.manager.service.ObjectStoreService; import org.apache.hertzbeat.warehouse.service.WarehouseService; -import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.ObjectProvider; import org.springframework.context.annotation.Lazy; import org.springframework.context.event.EventListener; @@ -92,8 +91,7 @@ import static java.util.Objects.isNull; @Service @Order(value = Ordered.HIGHEST_PRECEDENCE) @Slf4j -public class AppServiceImpl implements AppService, MonitorDefinitionSourceReader, MonitorDefinitionCommandExecutor, - InitializingBean { +public class AppServiceImpl implements AppService, MonitorDefinitionSourceReader, MonitorDefinitionCommandExecutor { private static final String PUSH_PROTOCOL_METRICS_NAME = "metrics"; private static final String[] RISKY_DEFINE_TOKENS = {"ScriptEngineManager", "URLClassLoader", "!!", @@ -605,8 +603,8 @@ public class AppServiceImpl implements AppService, MonitorDefinitionSourceReader } } - @Override - public void afterPropertiesSet() throws Exception { + /** Loads the definition store when the normal business runtime opens. */ + public void initializeRuntimeDefinitions() { // Guaranteed to be non-null due to constructor injection var objectStoreConfig = objectStoreConfigService.getConfig(); refreshStore(objectStoreConfig); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/ManagerBusinessRuntimeInitializer.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/ManagerBusinessRuntimeInitializer.java new file mode 100644 index 0000000000..6202c64bfd --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/ManagerBusinessRuntimeInitializer.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.service.impl; + +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; +import org.springframework.boot.CommandLineRunner; +import org.springframework.stereotype.Component; + +/** + * Initializes Manager business runtime state after setup completes. Object-store runtime must be ready before + * monitor definitions choose their backing store; stored plugin parameters must then precede plugin status and + * classloader convergence so loaded plugins see their persisted configuration. + */ +@Component +@ConditionalOnNormalBusinessRuntime +public final class ManagerBusinessRuntimeInitializer implements CommandLineRunner { + + private final ObjectStoreConfigServiceImpl objectStoreConfigService; + private final AppServiceImpl appService; + private final PluginParameterServiceImpl pluginParameterService; + private final PluginServiceImpl pluginService; + + public ManagerBusinessRuntimeInitializer(ObjectStoreConfigServiceImpl objectStoreConfigService, + AppServiceImpl appService, + PluginParameterServiceImpl pluginParameterService, + PluginServiceImpl pluginService) { + this.objectStoreConfigService = objectStoreConfigService; + this.appService = appService; + this.pluginParameterService = pluginParameterService; + this.pluginService = pluginService; + } + + @Override + public void run(String... args) { + objectStoreConfigService.initializeRuntimeState(); + appService.initializeRuntimeDefinitions(); + pluginParameterService.loadStoredParameters(); + pluginService.syncPluginStatus(); + pluginService.loadJarToClassLoader(); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/ObjectStoreConfigServiceImpl.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/ObjectStoreConfigServiceImpl.java index 88c596e516..bd549a62ec 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/ObjectStoreConfigServiceImpl.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/ObjectStoreConfigServiceImpl.java @@ -30,7 +30,6 @@ import org.apache.hertzbeat.manager.pojo.dto.ObjectStoreConfigResponse; import org.apache.hertzbeat.manager.pojo.dto.ObjectStoreDTO; import org.apache.hertzbeat.manager.service.ObjectStoreConfigMapper; import org.apache.hertzbeat.manager.service.ObjectStoreConfigService; -import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.support.DefaultListableBeanFactory; import org.springframework.context.ApplicationContext; import org.springframework.core.Ordered; @@ -50,7 +49,7 @@ import tools.jackson.core.type.TypeReference; @Service public class ObjectStoreConfigServiceImpl extends AbstractGeneralConfigServiceImpl> - implements InitializingBean, ObjectStoreConfigService { + implements ObjectStoreConfigService { private static final String BEAN_NAME = "ObjectStoreService"; @Resource @@ -160,8 +159,8 @@ public class ObjectStoreConfigServiceImpl extends mapper.validateObsEndpoint(endpoint); } - @Override - public void afterPropertiesSet() throws Exception { + /** Applies the persisted object-store configuration when the normal business runtime opens. */ + public void initializeRuntimeState() { applyRuntime(getConfig()); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/PluginParameterServiceImpl.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/PluginParameterServiceImpl.java index 6653947414..8351e5af94 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/PluginParameterServiceImpl.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/PluginParameterServiceImpl.java @@ -17,7 +17,6 @@ package org.apache.hertzbeat.manager.service.impl; -import jakarta.annotation.PostConstruct; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -71,7 +70,6 @@ public class PluginParameterServiceImpl implements PluginParameterService { private final AfterCommitPublisher afterCommitPublisher; - @PostConstruct void loadStoredParameters() { try { Map> grouped = pluginParamDao.findAll().stream() diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/PluginServiceImpl.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/PluginServiceImpl.java index 0cdc958b6a..1e6201db3a 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/PluginServiceImpl.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/PluginServiceImpl.java @@ -17,7 +17,6 @@ package org.apache.hertzbeat.manager.service.impl; -import jakarta.annotation.PostConstruct; import jakarta.persistence.criteria.Predicate; import java.io.File; import java.io.FileOutputStream; @@ -375,8 +374,7 @@ public class PluginServiceImpl implements PluginService { /** * Load all plugin enabled states into memory */ - @PostConstruct - private void syncPluginStatus() { + void syncPluginStatus() { List plugins = metadataDao.findAll(); Map statusMap = new HashMap<>(); Map itemToPluginMetadataIdMap = new HashMap<>(); @@ -405,8 +403,7 @@ public class PluginServiceImpl implements PluginService { /** * load jar to classloader */ - @PostConstruct - private void loadJarToClassLoader() { + void loadJarToClassLoader() { pluginClassLoaderLock.writeLock().lock(); try { try { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java index 8b5818ffd7..073e86abbd 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java @@ -233,6 +233,7 @@ public final class SetupApiContract { SETUP_CODE_INVALID("setup_code_invalid"), SETUP_CODE_EXPIRED("setup_code_expired"), SETUP_RATE_LIMITED("setup_rate_limited"), + SETUP_NOT_COMPLETE("setup_not_complete"), CONFIG_READ_ONLY("config_read_only"), CONFIG_WRITE_FAILED("config_write_failed"), CONFIG_RECOVERY_REQUIRED("config_recovery_required"), diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeAccessConfiguration.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeAccessConfiguration.java new file mode 100644 index 0000000000..4cd9442a64 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeAccessConfiguration.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.runtime; + +import org.apache.hertzbeat.common.runtime.BusinessRuntimeGate; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.web.servlet.FilterRegistrationBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.Ordered; + +/** Registers the runtime boundary before authentication and business filters. */ +@Configuration(proxyBeanMethods = false) +public class SetupRuntimeAccessConfiguration { + + @Bean + @ConditionalOnMissingBean + public SetupRuntimeAccessFilter setupRuntimeAccessFilter(BusinessRuntimeGate gate) { + return new SetupRuntimeAccessFilter(gate); + } + + @Bean + public FilterRegistrationBean setupRuntimeAccessFilterRegistration( + SetupRuntimeAccessFilter filter) { + FilterRegistrationBean registration = new FilterRegistrationBean<>(filter); + registration.setOrder(Ordered.HIGHEST_PRECEDENCE + 1); + registration.addUrlPatterns("/*"); + return registration; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeAccessFilter.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeAccessFilter.java new file mode 100644 index 0000000000..26d0151637 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeAccessFilter.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.runtime; + +import static org.apache.hertzbeat.common.constants.CommonConstants.FAIL_CODE; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Set; +import org.apache.hertzbeat.common.entity.dto.Message; +import org.apache.hertzbeat.common.runtime.BusinessRuntimeGate; +import org.apache.hertzbeat.common.util.JsonUtil; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.web.filter.OncePerRequestFilter; + +/** Blocks business HTTP access while the full context is setup-gated. */ +public final class SetupRuntimeAccessFilter extends OncePerRequestFilter { + + private static final Set HEALTH_PATHS = Set.of( + "/actuator/health", "/actuator/health/liveness", "/actuator/health/readiness"); + private final BusinessRuntimeGate gate; + + public SetupRuntimeAccessFilter(BusinessRuntimeGate gate) { + this.gate = gate; + } + + @Override + protected boolean shouldNotFilter(HttpServletRequest request) { + if (gate.isOpen()) { + return true; + } + String path = request.getServletPath(); + if (path == null || path.isEmpty()) { + path = pathWithinApplication(request); + } + if (path.startsWith("/api/setup/") || path.startsWith("/setup/") || HEALTH_PATHS.contains(path)) { + return true; + } + return !path.startsWith("/api/") && !path.startsWith("/actuator/"); + } + + private String pathWithinApplication(HttpServletRequest request) { + String requestUri = request.getRequestURI(); + String contextPath = request.getContextPath(); + if (contextPath != null && !contextPath.isEmpty() && requestUri.startsWith(contextPath)) { + return requestUri.substring(contextPath.length()); + } + return requestUri; + } + + @Override + protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain chain) + throws IOException, ServletException { + response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); + response.setHeader(HttpHeaders.CACHE_CONTROL, "no-store"); + response.setCharacterEncoding(StandardCharsets.UTF_8.name()); + response.setContentType(MediaType.APPLICATION_JSON_VALUE); + response.getWriter().write(JsonUtil.toJson( + Message.fail(FAIL_CODE, SetupErrorCode.SETUP_NOT_COMPLETE.value()))); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransition.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransition.java new file mode 100644 index 0000000000..61b31645b2 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransition.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.runtime; + +/** + * Application boundary used by setup completion to activate the normal runtime. + * + *

The transition closes the currently active Spring context. Callers must therefore invoke it from the + * asynchronous setup operation after the HTTP response has been committed, rather than from the request thread. + */ +@FunctionalInterface +public interface SetupRuntimeTransition { + + void completeSetup(); +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/status/CalculateStatusTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/status/CalculateStatusTest.java index 0ec7833213..2e607cd84a 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/status/CalculateStatusTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/status/CalculateStatusTest.java @@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.verifyNoInteractions; import java.util.Collections; import java.util.concurrent.CountDownLatch; @@ -63,7 +64,50 @@ class CalculateStatusTest { @BeforeEach void setUp() { calculateStatus = new CalculateStatus(statusPageOrgDao, statusPageComponentDao, statusProperties(), - statusPageHistoryDao, monitorDao, new VirtualThreadProperties(), false); + statusPageHistoryDao, monitorDao, new VirtualThreadProperties()); + } + + @Test + void constructorIsPassive() { + assertFalse(calculateStatus.isStarted()); + verifyNoInteractions(statusPageOrgDao, statusPageComponentDao, statusPageHistoryDao, monitorDao); + } + + @Test + void lifecycleIsIdempotent() { + calculateStatus.start(); + calculateStatus.start(); + + assertTrue(calculateStatus.isStarted()); + + calculateStatus.destroy(); + calculateStatus.destroy(); + + assertFalse(calculateStatus.isStarted()); + verifyNoInteractions(statusPageOrgDao, statusPageComponentDao, statusPageHistoryDao, monitorDao); + } + + @Test + void disabledVirtualThreadsStillReachStartedState() { + calculateStatus.destroy(); + calculateStatus = new CalculateStatus(statusPageOrgDao, statusPageComponentDao, statusProperties(), + statusPageHistoryDao, monitorDao, + new VirtualThreadProperties(false, null, null, null, null, null, null)); + + calculateStatus.start(); + + assertTrue(calculateStatus.isStarted()); + } + + @Test + void dispatchAfterDestroyIsSafeNoOp() { + calculateStatus.start(); + calculateStatus.destroy(); + + calculateStatus.dispatchCalculate(); + calculateStatus.dispatchCombineHistory(); + + verifyNoInteractions(statusPageOrgDao, statusPageComponentDao, statusPageHistoryDao, monitorDao); } @AfterEach @@ -75,6 +119,7 @@ class CalculateStatusTest { @Test void dispatchCalculateRunsOnVirtualThread() throws Exception { + calculateStatus.start(); CountDownLatch latch = new CountDownLatch(1); AtomicBoolean virtualThread = new AtomicBoolean(false); org.mockito.Mockito.doAnswer(invocation -> { @@ -91,6 +136,7 @@ class CalculateStatusTest { @Test void dispatchCombineHistoryRunsOnVirtualThread() throws Exception { + calculateStatus.start(); CountDownLatch latch = new CountDownLatch(1); AtomicBoolean virtualThread = new AtomicBoolean(false); org.mockito.Mockito.doAnswer(invocation -> { @@ -107,6 +153,7 @@ class CalculateStatusTest { @Test void dispatchCalculateDoesNotRunConcurrently() throws Exception { + calculateStatus.start(); CountDownLatch firstStarted = new CountDownLatch(1); CountDownLatch releaseFirst = new CountDownLatch(1); CountDownLatch secondStarted = new CountDownLatch(1); @@ -138,6 +185,41 @@ class CalculateStatusTest { assertEquals(1, maxConcurrent.get()); } + @Test + void destroyWhileCalculateIsRunningDropsPendingDispatch() throws Exception { + calculateStatus.start(); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch interrupted = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger invocations = new AtomicInteger(); + org.mockito.Mockito.doAnswer(invocation -> { + int current = invocations.incrementAndGet(); + if (current == 1) { + started.countDown(); + try { + Thread.sleep(5000L); + } catch (InterruptedException e) { + interrupted.countDown(); + Thread.currentThread().interrupt(); + } + } else { + secondStarted.countDown(); + } + return Collections.emptyList(); + }).when(statusPageOrgDao).findAll(); + + calculateStatus.dispatchCalculate(); + assertTrue(started.await(5, TimeUnit.SECONDS)); + calculateStatus.dispatchCalculate(); + + calculateStatus.destroy(); + calculateStatus.dispatchCalculate(); + + assertTrue(interrupted.await(5, TimeUnit.SECONDS)); + assertFalse(secondStarted.await(500, TimeUnit.MILLISECONDS)); + assertEquals(1, invocations.get()); + } + private StatusProperties statusProperties() { StatusProperties statusProperties = new StatusProperties(); StatusProperties.CalculateProperties calculateProperties = new StatusProperties.CalculateProperties(); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/monitor/definition/MonitorDefinitionCommandPortTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/monitor/definition/MonitorDefinitionCommandPortTest.java index 81da2acbfa..d150008985 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/monitor/definition/MonitorDefinitionCommandPortTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/monitor/definition/MonitorDefinitionCommandPortTest.java @@ -99,7 +99,7 @@ class MonitorDefinitionCommandPortTest { monitorServiceProvider, objectStoreServiceProvider); commandService = new MonitorDefinitionCommandService(appService); - appService.afterPropertiesSet(); + appService.initializeRuntimeDefinitions(); clearInvocations(defineDao, monitorDao, monitorService); } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServerTest.java index cbc9e46557..9649fa7c12 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServerTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServerTest.java @@ -19,11 +19,14 @@ package org.apache.hertzbeat.manager.scheduler.netty; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import io.netty.channel.Channel; @@ -36,6 +39,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.hertzbeat.alert.calculate.CollectorAlertHandler; import org.apache.hertzbeat.common.concurrent.BackgroundTaskExecutor; import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.apache.hertzbeat.common.entity.message.ClusterMsg; import org.apache.hertzbeat.common.queue.CommonDataQueue; import org.apache.hertzbeat.manager.scheduler.CollectorJobScheduler; import org.apache.hertzbeat.manager.scheduler.SchedulerProperties; @@ -79,6 +83,35 @@ class ManageServerTest { ReflectionTestUtils.setField(manageServer, "remotingServer", mock(RemotingServer.class)); } + @Test + void commandsBeforeStartFailSafely() { + ManageServer inactiveServer = new ManageServer(schedulerProperties(), collectorJobScheduler, commonThreadPool, + collectorAlertHandler, commonDataQueue, new VirtualThreadProperties(), runtimeStatusRegistry); + ClusterMsg.Message message = ClusterMsg.Message.getDefaultInstance(); + clearInvocations(collectorJobScheduler, runtimeStatusRegistry); + + assertFalse(inactiveServer.sendMsg("collector-1", message)); + assertNull(inactiveServer.sendMsgSync("collector-1", message)); + inactiveServer.closeChannel("collector-1"); + + verifyNoInteractions(collectorJobScheduler, runtimeStatusRegistry); + } + + @Test + void commandsAfterShutdownFailSafely() { + Channel channel = mock(Channel.class); + clientChannelTable().put("collector-1", channel); + ClusterMsg.Message message = ClusterMsg.Message.getDefaultInstance(); + clearInvocations(collectorJobScheduler, runtimeStatusRegistry); + manageServer.shutdown(); + + assertFalse(manageServer.sendMsg("collector-1", message)); + assertNull(manageServer.sendMsgSync("collector-1", message)); + manageServer.closeChannel("collector-1"); + + verifyNoInteractions(channel, collectorJobScheduler, runtimeStatusRegistry); + } + @AfterEach void tearDown() { if (manageServer != null) { @@ -144,6 +177,43 @@ class ManageServerTest { assertEquals(1, maxConcurrent.get()); } + @Test + void shutdownWhileCheckIsRunningDropsPendingWorkAndIsIdempotent() throws Exception { + Channel channel = mock(Channel.class); + clientChannelTable().put("collector-1", channel); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch interrupted = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger invocations = new AtomicInteger(); + org.mockito.Mockito.doAnswer(invocation -> { + int current = invocations.incrementAndGet(); + if (current == 1) { + started.countDown(); + try { + Thread.sleep(5000L); + } catch (InterruptedException e) { + interrupted.countDown(); + Thread.currentThread().interrupt(); + } + } else { + secondStarted.countDown(); + } + return true; + }).when(channel).isActive(); + + manageServer.dispatchChannelHealthCheck(); + assertTrue(started.await(5, TimeUnit.SECONDS)); + manageServer.dispatchChannelHealthCheck(); + + manageServer.shutdown(); + manageServer.shutdown(); + manageServer.dispatchChannelHealthCheck(); + + assertTrue(interrupted.await(5, TimeUnit.SECONDS)); + assertFalse(secondStarted.await(500, TimeUnit.MILLISECONDS)); + assertEquals(1, invocations.get()); + } + @SuppressWarnings("unchecked") private Map clientChannelTable() { return (Map) ReflectionTestUtils.getField(manageServer, "clientChannelTable"); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/AppServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/AppServiceTest.java index 8347badb41..2fb6640290 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/AppServiceTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/AppServiceTest.java @@ -102,7 +102,7 @@ class AppServiceTest { @BeforeEach void setUp() throws Exception { when(defineDao.findAll()).thenReturn(new ArrayList<>()); - appService.afterPropertiesSet(); + appService.initializeRuntimeDefinitions(); } @Test diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/ObjectStoreConfigServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/ObjectStoreConfigServiceTest.java index 7a2b8aea66..7e327582e7 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/ObjectStoreConfigServiceTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/ObjectStoreConfigServiceTest.java @@ -78,7 +78,7 @@ class ObjectStoreConfigServiceTest { @Test void testStartupWithNullConfigDoesNotPublishChange() throws Exception { - objectStoreConfigService.afterPropertiesSet(); + objectStoreConfigService.initializeRuntimeState(); verify(ctx, never()).publishEvent(any()); } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java index 8728f4fbf7..45f93a2241 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java @@ -163,7 +163,7 @@ class SetupApiContractTest { @Test void freezesStableSafeErrorCodes() throws Exception { assertWireValues(SetupErrorCode.values(), "setup_complete", "setup_locked", "setup_code_invalid", - "setup_code_expired", "setup_rate_limited", "config_read_only", "config_write_failed", + "setup_code_expired", "setup_rate_limited", "setup_not_complete", "config_read_only", "config_write_failed", "config_recovery_required", "metadata_connection_failed", "metadata_kind_unsupported", "metadata_schema_mismatch", "metadata_insufficient_privileges", "telemetry_connection_failed", "public_address_invalid", "mail_connection_failed", "administrator_already_configured", diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeAccessFilterTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeAccessFilterTest.java new file mode 100644 index 0000000000..af58474bc8 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeAccessFilterTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.runtime; + +import static org.apache.hertzbeat.common.constants.CommonConstants.FAIL_CODE; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import jakarta.servlet.FilterChain; +import java.util.List; +import org.apache.hertzbeat.common.runtime.BusinessRuntimeGate; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.junit.jupiter.api.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; + +class SetupRuntimeAccessFilterTest { + + @Test + void gatedRuntimeAllowsSetupHealthAndStaticApplicationPaths() throws Exception { + SetupRuntimeAccessFilter filter = new SetupRuntimeAccessFilter( + BusinessRuntimeGate.fixed(RuntimeMode.FULL_SETUP_GATED)); + List allowed = List.of( + "/api/setup/status", + "/api/setup/operations/op-1", + "/actuator/health", + "/actuator/health/liveness", + "/actuator/health/readiness", + "/setup/index.html"); + for (String path : allowed) { + MockHttpServletRequest request = new MockHttpServletRequest("GET", path); + MockHttpServletResponse response = new MockHttpServletResponse(); + boolean[] invoked = {false}; + filter.doFilter(request, response, invokedChain(invoked)); + assertTrue(invoked[0], path); + } + } + + @Test + void gatedRuntimeRejectsBusinessApiBehindServletContextPath() throws Exception { + SetupRuntimeAccessFilter filter = new SetupRuntimeAccessFilter( + BusinessRuntimeGate.fixed(RuntimeMode.FULL_SETUP_GATED)); + MockHttpServletRequest request = new MockHttpServletRequest("GET", "/hertzbeat/api/summary"); + request.setContextPath("/hertzbeat"); + request.setServletPath("/api/summary"); + MockHttpServletResponse response = new MockHttpServletResponse(); + boolean[] invoked = {false}; + + filter.doFilter(request, response, invokedChain(invoked)); + + assertFalse(invoked[0]); + assertEquals(503, response.getStatus()); + } + + @Test + void gatedRuntimeRejectsReadsWritesAndOtlpWithSafeNoStoreEnvelope() throws Exception { + SetupRuntimeAccessFilter filter = new SetupRuntimeAccessFilter( + BusinessRuntimeGate.fixed(RuntimeMode.FULL_SETUP_GATED)); + for (String path : List.of("/api/summary", "/api/monitor", "/api/otlp/v1/metrics")) { + MockHttpServletRequest request = new MockHttpServletRequest("POST", path); + MockHttpServletResponse response = new MockHttpServletResponse(); + boolean[] invoked = {false}; + filter.doFilter(request, response, invokedChain(invoked)); + assertFalse(invoked[0], path); + assertEquals(503, response.getStatus()); + assertEquals("no-store", response.getHeader("Cache-Control")); + assertTrue(response.getContentType().startsWith("application/json")); + String body = response.getContentAsString(); + assertTrue(body.contains("\"code\":" + FAIL_CODE), body); + assertTrue(body.contains("\"msg\":\"setup_not_complete\""), body); + assertFalse(body.contains("Exception"), body); + } + } + + @Test + void gatedRuntimeDoesNotExposeOtherActuatorEndpoints() throws Exception { + SetupRuntimeAccessFilter filter = new SetupRuntimeAccessFilter( + BusinessRuntimeGate.fixed(RuntimeMode.FULL_SETUP_GATED)); + MockHttpServletRequest request = new MockHttpServletRequest("GET", "/actuator/env"); + MockHttpServletResponse response = new MockHttpServletResponse(); + boolean[] invoked = {false}; + filter.doFilter(request, response, invokedChain(invoked)); + assertFalse(invoked[0]); + assertEquals(503, response.getStatus()); + } + + @Test + void normalRuntimeDoesNotInterceptBusinessApi() throws Exception { + SetupRuntimeAccessFilter filter = new SetupRuntimeAccessFilter(BusinessRuntimeGate.fixed(RuntimeMode.NORMAL)); + MockHttpServletRequest request = new MockHttpServletRequest("POST", "/api/monitor"); + MockHttpServletResponse response = new MockHttpServletResponse(); + boolean[] invoked = {false}; + filter.doFilter(request, response, invokedChain(invoked)); + assertTrue(invoked[0]); + } + + private FilterChain invokedChain(boolean[] invoked) { + return (request, response) -> invoked[0] = true; + } +} diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/config/OpenTelemetryLogbackAppenderInstaller.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/config/OpenTelemetryLogbackAppenderInstaller.java index d3a91ada72..52fc724ed1 100644 --- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/config/OpenTelemetryLogbackAppenderInstaller.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/config/OpenTelemetryLogbackAppenderInstaller.java @@ -21,6 +21,7 @@ import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.instrumentation.logback.appender.v1_0.OpenTelemetryAppender; import io.opentelemetry.sdk.OpenTelemetrySdk; import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.springframework.beans.factory.InitializingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; @@ -29,6 +30,7 @@ import org.springframework.stereotype.Component; * Installs the OpenTelemetryAppender for Logback once the auto-configured SDK is ready. */ @Component +@ConditionalOnNormalBusinessRuntime @ConditionalOnProperty(name = "warehouse.store.greptime.enabled", havingValue = "true") @Slf4j public class OpenTelemetryLogbackAppenderInstaller implements InitializingBean { diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/config/OtlpGrpcServerConfig.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/config/OtlpGrpcServerConfig.java index 68060f7c47..285680928a 100644 --- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/config/OtlpGrpcServerConfig.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/config/OtlpGrpcServerConfig.java @@ -39,6 +39,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hertzbeat.common.observability.gateway.AuthTokenRequestContext; import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes; import org.apache.hertzbeat.common.observability.gateway.ObservabilityAccessTokenGateway; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.apache.hertzbeat.observability.ingestion.grpc.OtlpGrpcLogsService; import org.apache.hertzbeat.observability.ingestion.grpc.OtlpGrpcMetricsService; import org.apache.hertzbeat.observability.ingestion.grpc.OtlpGrpcTraceService; @@ -51,6 +52,7 @@ import org.springframework.context.annotation.Configuration; * OTLP gRPC server configuration. */ @Configuration +@ConditionalOnNormalBusinessRuntime public class OtlpGrpcServerConfig { @Bean(initMethod = "start", destroyMethod = "stop") diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeApmFlowInitializer.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeApmFlowInitializer.java index 933aa49ec9..6ab86e13cd 100644 --- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeApmFlowInitializer.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeApmFlowInitializer.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Set; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.apache.hertzbeat.observability.ingestion.retry.OtlpIngestionRetryService; import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties; import org.springframework.beans.factory.ObjectProvider; @@ -51,6 +52,7 @@ import org.springframework.web.util.UriUtils; */ @Slf4j @Component +@ConditionalOnNormalBusinessRuntime @ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true") public class GreptimeApmFlowInitializer { diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeLogPipelineInitializer.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeLogPipelineInitializer.java index f44340a149..720fcb3c38 100644 --- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeLogPipelineInitializer.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeLogPipelineInitializer.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.apache.hertzbeat.observability.ingestion.retry.OtlpIngestionRetryService; import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties; import org.springframework.beans.factory.ObjectProvider; @@ -50,6 +51,7 @@ import org.springframework.web.client.RestTemplate; */ @Slf4j @Component +@ConditionalOnNormalBusinessRuntime @ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true") public class GreptimeLogPipelineInitializer { diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeTraceTableInitializer.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeTraceTableInitializer.java index 61104c98fc..8f0ffb17b7 100644 --- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeTraceTableInitializer.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeTraceTableInitializer.java @@ -22,6 +22,7 @@ import java.nio.charset.StandardCharsets; import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.apache.hertzbeat.observability.ingestion.retry.OtlpIngestionRetryService; import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties; import org.springframework.beans.factory.ObjectProvider; @@ -46,6 +47,7 @@ import org.springframework.web.util.UriUtils; */ @Slf4j @Component +@ConditionalOnNormalBusinessRuntime @ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true") public class GreptimeTraceTableInitializer { diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/logs/sse/LogSseManager.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/logs/sse/LogSseManager.java index e64ea97841..9fc19a5330 100644 --- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/logs/sse/LogSseManager.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/logs/sse/LogSseManager.java @@ -61,27 +61,34 @@ public class LogSseManager { private final Map emitters = new ConcurrentHashMap<>(); private final Queue logQueue = new ConcurrentLinkedQueue<>(); private final Object queueLock = new Object(); - private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(r -> { - Thread t = new Thread(r, "sse-batch-scheduler"); - t.setDaemon(true); - return t; - }); + private ScheduledExecutorService scheduler; private final AtomicLong queueSize = new AtomicLong(0); private final AtomicLong broadcastSequence = new AtomicLong(0); private final AtomicBoolean closed = new AtomicBoolean(false); private final AtomicReference pendingGap = new AtomicReference<>(); - public LogSseManager() { + synchronized void start() { + if (scheduler != null) { + return; + } + scheduler = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "sse-batch-scheduler"); + t.setDaemon(true); + return t; + }); scheduler.scheduleAtFixedRate(this::flushBatch, BATCH_INTERVAL_MS, BATCH_INTERVAL_MS, TimeUnit.MILLISECONDS); scheduler.scheduleAtFixedRate( this::sendHeartbeats, HEARTBEAT_INTERVAL_MS, HEARTBEAT_INTERVAL_MS, TimeUnit.MILLISECONDS); } @PreDestroy - public void shutdown() { + public synchronized void shutdown() { if (!closed.compareAndSet(false, true)) { return; } + if (scheduler == null) { + return; + } scheduler.shutdownNow(); List subscribers = new ArrayList<>(emitters.values()); emitters.clear(); @@ -107,6 +114,7 @@ public class LogSseManager { queueSize.set(0); pendingGap.set(null); } + scheduler = null; } /** diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/logs/sse/LogSseManagerLifecycle.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/logs/sse/LogSseManagerLifecycle.java new file mode 100644 index 0000000000..a780773a9a --- /dev/null +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/logs/sse/LogSseManagerLifecycle.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.observability.logs.sse; + +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; +import org.springframework.boot.CommandLineRunner; +import org.springframework.stereotype.Component; + +/** Starts observability log SSE delivery only in normal runtime. */ +@Component("observabilityLogSseManagerLifecycle") +@ConditionalOnNormalBusinessRuntime +public final class LogSseManagerLifecycle implements CommandLineRunner { + + private final LogSseManager manager; + + public LogSseManagerLifecycle(LogSseManager manager) { + this.manager = manager; + } + + @Override + public void run(String... args) { + manager.start(); + } +} diff --git a/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/logs/sse/LogSseManagerTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/logs/sse/LogSseManagerTest.java index 21fd1e67f1..13fe1252a2 100644 --- a/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/logs/sse/LogSseManagerTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/logs/sse/LogSseManagerTest.java @@ -73,6 +73,7 @@ class LogSseManagerTest { @BeforeEach void setUp() { logSseManager = new LogSseManager(); + logSseManager.start(); } @AfterEach diff --git a/hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/OpenTelemetryLogbackAppenderInstaller.java b/hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/OpenTelemetryLogbackAppenderInstaller.java index 487b199876..de17089222 100644 --- a/hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/OpenTelemetryLogbackAppenderInstaller.java +++ b/hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/OpenTelemetryLogbackAppenderInstaller.java @@ -22,6 +22,7 @@ import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.instrumentation.logback.appender.v1_0.OpenTelemetryAppender; import io.opentelemetry.sdk.OpenTelemetrySdk; import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -32,6 +33,7 @@ import org.springframework.stereotype.Component; * OpenTelemetry SDK is available and GrepTimeDB integration is enabled. */ @Component +@ConditionalOnNormalBusinessRuntime @ConditionalOnProperty(name = "warehouse.store.greptime.enabled", havingValue = "true") @Slf4j public class OpenTelemetryLogbackAppenderInstaller implements InitializingBean { @@ -55,4 +57,4 @@ public class OpenTelemetryLogbackAppenderInstaller implements InitializingBean { this.openTelemetry != null ? this.openTelemetry.getClass().getName() : "null"); } } -} \ No newline at end of file +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/bootstrap/SetupOnlyApplication.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/bootstrap/SetupOnlyApplication.java new file mode 100644 index 0000000000..c2ce5800fa --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/bootstrap/SetupOnlyApplication.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.bootstrap; + +import org.apache.hertzbeat.common.runtime.BusinessRuntimeConfiguration; +import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeAccessConfiguration; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.actuate.autoconfigure.endpoint.EndpointAutoConfiguration; +import org.springframework.boot.actuate.autoconfigure.endpoint.web.WebEndpointAutoConfiguration; +import org.springframework.boot.autoconfigure.ImportAutoConfiguration; +import org.springframework.boot.health.autoconfigure.actuate.endpoint.HealthEndpointAutoConfiguration; +import org.springframework.boot.jackson.autoconfigure.JacksonAutoConfiguration; +import org.springframework.boot.tomcat.autoconfigure.servlet.TomcatServletWebServerAutoConfiguration; +import org.springframework.boot.webmvc.autoconfigure.DispatcherServletAutoConfiguration; +import org.springframework.boot.webmvc.autoconfigure.WebMvcAutoConfiguration; +import org.springframework.boot.webmvc.autoconfigure.actuate.endpoint.web.WebMvcHealthEndpointExtensionAutoConfiguration; +import org.springframework.boot.webmvc.autoconfigure.error.ErrorMvcAutoConfiguration; +import org.springframework.context.annotation.Import; + +/** Minimal source set used when full persistence configuration cannot safely start. */ +@SpringBootConfiguration +@ImportAutoConfiguration({ + JacksonAutoConfiguration.class, + TomcatServletWebServerAutoConfiguration.class, + DispatcherServletAutoConfiguration.class, + WebMvcAutoConfiguration.class, + ErrorMvcAutoConfiguration.class, + EndpointAutoConfiguration.class, + WebEndpointAutoConfiguration.class, + HealthEndpointAutoConfiguration.class, + WebMvcHealthEndpointExtensionAutoConfiguration.class +}) +@Import({BusinessRuntimeConfiguration.class, SetupRuntimeAccessConfiguration.class}) +public class SetupOnlyApplication { +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/HertzBeatApplication.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/HertzBeatApplication.java index ae6394357b..abc095c775 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/HertzBeatApplication.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/HertzBeatApplication.java @@ -18,12 +18,17 @@ package org.apache.hertzbeat.startup; import jakarta.annotation.PostConstruct; +import org.apache.hertzbeat.bootstrap.SetupOnlyApplication; import org.apache.hertzbeat.manager.nativex.HertzbeatRuntimeHintsRegistrar; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.apache.hertzbeat.startup.runtime.HertzBeatStartupCoordinator; +import org.apache.hertzbeat.startup.runtime.SpringStartupContextLauncher; +import org.apache.hertzbeat.startup.runtime.StartupModePropertyProbe; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.context.properties.ConfigurationPropertiesScan; import org.springframework.boot.persistence.autoconfigure.EntityScan; import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.FilterType; import org.springframework.context.annotation.ImportRuntimeHints; import org.springframework.data.jpa.repository.config.EnableJpaAuditing; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; @@ -34,11 +39,13 @@ import org.springframework.scheduling.annotation.EnableScheduling; * HertzBeat main application startup class. * This class replaces the original Manager class as the main entry point for HertzBeat application. */ -@SpringBootApplication +@SpringBootConfiguration +@EnableAutoConfiguration @EnableJpaAuditing @EnableJpaRepositories(basePackages = {"org.apache.hertzbeat"}) @EntityScan(basePackages = {"org.apache.hertzbeat"}) -@ComponentScan(basePackages = {"org.apache.hertzbeat"}) +@ComponentScan(basePackages = {"org.apache.hertzbeat"}, excludeFilters = @ComponentScan.Filter( + type = FilterType.ASSIGNABLE_TYPE, classes = SetupOnlyApplication.class)) @ConfigurationPropertiesScan(basePackages = {"org.apache.hertzbeat"}) @ImportRuntimeHints(HertzbeatRuntimeHintsRegistrar.class) @EnableAsync @@ -46,7 +53,10 @@ import org.springframework.scheduling.annotation.EnableScheduling; public class HertzBeatApplication { public static void main(String[] args) { - SpringApplication.run(HertzBeatApplication.class, args); + SpringStartupContextLauncher launcher = new SpringStartupContextLauncher(); + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( + new StartupModePropertyProbe(), launcher); + coordinator.start(args); } @PostConstruct diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java new file mode 100644 index 0000000000..440a23e07f --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup.runtime; + +import java.util.Objects; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; + +/** Serializes setup-to-normal transitions and always closes the old context first. */ +public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition { + + private final StartupDecisionProbe probe; + private final StartupContextLauncher launcher; + private String[] args = new String[0]; + private RunningApplicationContext currentContext; + + public HertzBeatStartupCoordinator(StartupDecisionProbe probe, StartupContextLauncher launcher) { + this.probe = Objects.requireNonNull(probe, "probe"); + this.launcher = Objects.requireNonNull(launcher, "launcher"); + } + + public synchronized RunningApplicationContext start(String[] applicationArgs) { + args = applicationArgs == null ? new String[0] : applicationArgs.clone(); + StartupDecision decision; + try { + decision = Objects.requireNonNull(probe.probe(), "startup decision"); + } catch (RuntimeException exception) { + decision = StartupDecision.recovery(); + } + return transition(decision); + } + + @Override + public synchronized void completeSetup() { + transition(new StartupDecision(RuntimeMode.NORMAL, SetupPhase.COMPLETE, null)); + } + + public synchronized RunningApplicationContext transition(StartupDecision decision) { + Objects.requireNonNull(decision, "decision"); + if (currentContext != null && currentContext.isActive() && currentContext.mode() == decision.mode()) { + return currentContext; + } + closeCurrent(); + try { + currentContext = launch(decision); + } catch (RuntimeException launchFailure) { + if (decision.mode() == RuntimeMode.RECOVERY) { + throw launchFailure; + } + try { + currentContext = launch(StartupDecision.recovery()); + } catch (RuntimeException recoveryFailure) { + recoveryFailure.addSuppressed(launchFailure); + throw recoveryFailure; + } + } + return currentContext; + } + + public synchronized RuntimeMode mode() { + return currentContext == null ? null : currentContext.mode(); + } + + public synchronized RunningApplicationContext currentContext() { + return currentContext; + } + + private void closeCurrent() { + if (currentContext != null) { + currentContext.close(); + currentContext = null; + } + } + + private RunningApplicationContext launch(StartupDecision decision) { + return Objects.requireNonNull( + launcher.launch(decision, args.clone(), this), + "startup context launcher returned null for " + decision.mode().value()); + } +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/RunningApplicationContext.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/RunningApplicationContext.java new file mode 100644 index 0000000000..a955187b32 --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/RunningApplicationContext.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup.runtime; + +import org.apache.hertzbeat.common.runtime.RuntimeMode; + +/** Lifecycle boundary that prevents two HertzBeat contexts from running concurrently. */ +public interface RunningApplicationContext extends AutoCloseable { + + RuntimeMode mode(); + + boolean isActive(); + + @Override + void close(); +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/SpringStartupContextLauncher.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/SpringStartupContextLauncher.java new file mode 100644 index 0000000000..9aeaf6c270 --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/SpringStartupContextLauncher.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup.runtime; + +import java.util.Map; +import org.apache.hertzbeat.bootstrap.SetupOnlyApplication; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; +import org.apache.hertzbeat.startup.HertzBeatApplication; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.core.env.MapPropertySource; + +/** Spring implementation with explicit AOT-visible source classes. */ +public final class SpringStartupContextLauncher implements StartupContextLauncher { + + @Override + public RunningApplicationContext launch( + StartupDecision decision, String[] args, SetupRuntimeTransition setupRuntimeTransition) { + ConfigurableApplicationContext context = launchSpringContext(decision, args, setupRuntimeTransition); + return new SpringRunningApplicationContext(decision.mode(), context); + } + + ConfigurableApplicationContext launchSpringContext( + StartupDecision decision, String[] args, SetupRuntimeTransition setupRuntimeTransition) { + return new SpringApplicationBuilder(sourceFor(decision.mode())) + .initializers(context -> { + context.getEnvironment().getPropertySources().addFirst( + new MapPropertySource("hertzbeatInternalRuntimeMode", + Map.of(RuntimeMode.PROPERTY_NAME, decision.mode().value()))); + context.getBeanFactory().registerSingleton( + "setupRuntimeTransition", setupRuntimeTransition); + }) + .run(args); + } + + static Class sourceFor(RuntimeMode mode) { + return mode == RuntimeMode.NORMAL || mode == RuntimeMode.FULL_SETUP_GATED + ? HertzBeatApplication.class : SetupOnlyApplication.class; + } + + private record SpringRunningApplicationContext(RuntimeMode mode, ConfigurableApplicationContext delegate) + implements RunningApplicationContext { + + @Override + public boolean isActive() { + return delegate.isActive(); + } + + @Override + public void close() { + delegate.close(); + } + } +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupContextLauncher.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupContextLauncher.java new file mode 100644 index 0000000000..7ed1882a02 --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupContextLauncher.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup.runtime; + +import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; + +/** Opens exactly one source set for a classified runtime mode. */ +@FunctionalInterface +public interface StartupContextLauncher { + + RunningApplicationContext launch( + StartupDecision decision, String[] args, SetupRuntimeTransition setupRuntimeTransition); +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupDecision.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupDecision.java new file mode 100644 index 0000000000..7669714a1e --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupDecision.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup.runtime; + +import java.util.Objects; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; + +/** Safe startup classification produced before an application context is opened. */ +public record StartupDecision(RuntimeMode mode, SetupPhase phase, SetupErrorCode errorCode) { + + public StartupDecision { + Objects.requireNonNull(mode, "mode"); + Objects.requireNonNull(phase, "phase"); + } + + public static StartupDecision normal() { + return new StartupDecision(RuntimeMode.NORMAL, SetupPhase.COMPLETE, null); + } + + public static StartupDecision recovery() { + return new StartupDecision(RuntimeMode.RECOVERY, SetupPhase.RECOVERY_REQUIRED, + SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupDecisionProbe.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupDecisionProbe.java new file mode 100644 index 0000000000..ce2e5b535c --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupDecisionProbe.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup.runtime; + +/** Replaceable read-only source for the startup decision made before opening a context. */ +@FunctionalInterface +public interface StartupDecisionProbe { + + StartupDecision probe(); +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbe.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbe.java new file mode 100644 index 0000000000..74faf0e476 --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbe.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup.runtime; + +import java.util.Objects; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; + +/** Applies a local/container break-glass override before delegating to the installation probe. */ +public final class StartupModePropertyProbe implements StartupDecisionProbe { + + public static final String PROPERTY_NAME = "hertzbeat.startup.mode"; + public static final String ENVIRONMENT_NAME = "HERTZBEAT_STARTUP_MODE"; + + private final StartupDecisionProbe fallback; + + public StartupModePropertyProbe() { + this(StartupDecision::normal); + } + + public StartupModePropertyProbe(StartupDecisionProbe fallback) { + this.fallback = Objects.requireNonNull(fallback, "fallback"); + } + + @Override + public StartupDecision probe() { + return decide(System.getProperty(PROPERTY_NAME), System.getenv(ENVIRONMENT_NAME)); + } + + StartupDecision decide(String systemValue, String environmentValue) { + String value = selectConfiguredValue(systemValue, environmentValue); + return value == null ? fallback.probe() : decisionFor(value); + } + + static String selectConfiguredValue(String systemValue, String environmentValue) { + return systemValue == null ? environmentValue : systemValue; + } + + static StartupDecision decisionFor(String value) { + RuntimeMode mode = RuntimeMode.fromProperty(value); + return switch (mode) { + case NORMAL -> StartupDecision.normal(); + case SETUP_ONLY -> new StartupDecision(mode, SetupPhase.CONFIGURATION_REQUIRED, null); + case FULL_SETUP_GATED -> new StartupDecision(mode, SetupPhase.ADMINISTRATOR_REQUIRED, null); + case RECOVERY -> StartupDecision.recovery(); + }; + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinatorTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinatorTest.java new file mode 100644 index 0000000000..4b782e5bbb --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinatorTest.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup.runtime; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.ArrayList; +import java.util.List; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; +import org.junit.jupiter.api.Test; + +class HertzBeatStartupCoordinatorTest { + + @Test + void startsFromProbeAndClosesGatedContextBeforeOpeningNormalExactlyOnce() { + RecordingLauncher launcher = new RecordingLauncher(); + StartupDecision gated = new StartupDecision(RuntimeMode.FULL_SETUP_GATED, + SetupPhase.ADMINISTRATOR_REQUIRED, null); + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator(() -> gated, launcher); + + RunningApplicationContext first = coordinator.start(new String[]{"--server.port=0"}); + SetupRuntimeTransition transition = launcher.transitions.getFirst(); + transition.completeSetup(); + RunningApplicationContext normal = coordinator.currentContext(); + transition.completeSetup(); + RunningApplicationContext repeated = coordinator.currentContext(); + + assertEquals(List.of("open:full_setup_gated", "close:full_setup_gated", "open:normal"), launcher.events); + assertEquals(List.of(coordinator, coordinator), launcher.transitions); + assertSame(normal, repeated); + assertEquals(RuntimeMode.NORMAL, coordinator.mode()); + assertFalse(first.isActive()); + } + + @Test + void launchFailureClosesOldContextAndFallsBackToRecovery() { + RecordingLauncher launcher = new RecordingLauncher(); + launcher.failMode = RuntimeMode.FULL_SETUP_GATED; + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( + () -> new StartupDecision(RuntimeMode.SETUP_ONLY, SetupPhase.CONFIGURATION_REQUIRED, null), launcher); + coordinator.start(new String[0]); + + RunningApplicationContext recovery = coordinator.transition(new StartupDecision( + RuntimeMode.FULL_SETUP_GATED, SetupPhase.ADMINISTRATOR_REQUIRED, null)); + + assertEquals(List.of("open:setup_only", "close:setup_only", "open:full_setup_gated", "open:recovery"), + launcher.events); + assertEquals(RuntimeMode.RECOVERY, coordinator.mode()); + assertSame(recovery, coordinator.currentContext()); + } + + @Test + void probeFailureCannotBeMisclassifiedAsNewInstallation() { + RecordingLauncher launcher = new RecordingLauncher(); + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( + () -> { + throw new IllegalStateException("database unreachable"); + }, launcher); + + coordinator.start(new String[0]); + + assertEquals(List.of("open:recovery"), launcher.events); + assertEquals(RuntimeMode.RECOVERY, coordinator.mode()); + } + + @Test + void recoveryFailureRetainsOriginalLaunchFailureAsSuppressed() { + RecordingLauncher launcher = new RecordingLauncher(); + launcher.failMode = RuntimeMode.NORMAL; + launcher.failRecovery = true; + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator(StartupDecision::normal, launcher); + + IllegalStateException failure = assertThrows(IllegalStateException.class, + () -> coordinator.start(new String[0])); + + assertEquals("launch failed: recovery", failure.getMessage()); + assertEquals(1, failure.getSuppressed().length); + assertEquals("launch failed: normal", failure.getSuppressed()[0].getMessage()); + } + + @Test + void nullContextIsAnExplicitLaunchFailure() { + StartupContextLauncher launcher = (decision, args, transition) -> null; + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator(StartupDecision::normal, launcher); + + NullPointerException failure = assertThrows(NullPointerException.class, + () -> coordinator.start(new String[0])); + + assertEquals("startup context launcher returned null for recovery", failure.getMessage()); + assertEquals(1, failure.getSuppressed().length); + assertEquals("startup context launcher returned null for normal", failure.getSuppressed()[0].getMessage()); + } + + private static final class RecordingLauncher implements StartupContextLauncher { + + private final List events = new ArrayList<>(); + private final List transitions = new ArrayList<>(); + private RuntimeMode failMode; + private boolean failRecovery; + + @Override + public RunningApplicationContext launch( + StartupDecision decision, String[] args, SetupRuntimeTransition setupRuntimeTransition) { + events.add("open:" + decision.mode().value()); + transitions.add(setupRuntimeTransition); + if (decision.mode() == failMode || (decision.mode() == RuntimeMode.RECOVERY && failRecovery)) { + throw new IllegalStateException("launch failed: " + decision.mode().value()); + } + return new RunningApplicationContext() { + private boolean active = true; + + @Override + public RuntimeMode mode() { + return decision.mode(); + } + + @Override + public boolean isActive() { + return active; + } + + @Override + public void close() { + active = false; + events.add("close:" + decision.mode().value()); + } + }; + } + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbeTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbeTest.java new file mode 100644 index 0000000000..670ea1e875 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbeTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup.runtime; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.junit.jupiter.api.Test; + +class StartupModePropertyProbeTest { + + @Test + void missingOverridePreservesNormalStartup() { + StartupDecision decision = new StartupModePropertyProbe().decide(null, null); + + assertEquals(RuntimeMode.NORMAL, decision.mode()); + assertEquals(SetupPhase.COMPLETE, decision.phase()); + } + + @Test + void systemPropertyTakesPrecedenceOverEnvironment() { + StartupDecision decision = new StartupModePropertyProbe().decide("full_setup_gated", "setup_only"); + + assertEquals(RuntimeMode.FULL_SETUP_GATED, decision.mode()); + assertEquals(SetupPhase.ADMINISTRATOR_REQUIRED, decision.phase()); + } + + @Test + void environmentSelectsSetupOnlyWhenSystemPropertyIsMissing() { + StartupDecision decision = new StartupModePropertyProbe().decide(null, "setup_only"); + + assertEquals(RuntimeMode.SETUP_ONLY, decision.mode()); + assertEquals(SetupPhase.CONFIGURATION_REQUIRED, decision.phase()); + } + + @Test + void invalidOverrideFailsClosedForCoordinatorRecovery() { + StartupModePropertyProbe probe = new StartupModePropertyProbe(); + + assertThrows(IllegalArgumentException.class, () -> probe.decide("unsupported", null)); + } + + @Test + void missingOverrideDelegatesToInstallationProbe() { + StartupDecision expected = StartupDecision.recovery(); + StartupDecisionProbe fallback = () -> expected; + + assertEquals(expected, new StartupModePropertyProbe(fallback).decide(null, null)); + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupRuntimeBoundaryContextTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupRuntimeBoundaryContextTest.java new file mode 100644 index 0000000000..db3abaf09f --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupRuntimeBoundaryContextTest.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup.runtime; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.hertzbeat.common.runtime.BusinessRuntimeGate; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; +import org.junit.jupiter.api.Test; +import org.springframework.context.ConfigurableApplicationContext; + +class StartupRuntimeBoundaryContextTest { + + private static final SetupRuntimeTransition SETUP_RUNTIME_TRANSITION = () -> { }; + + private static final String[] SIDE_EFFECT_BEANS = { + "schedulerInit", "manageServerLifecycle", "periodicAlertRuleSchedulerLifecycle", "noticeTemplateInitializer", + "otlpGrpcServerConfig", "serviceDiscoveryWorker", + "openTelemetryLogbackAppenderInstaller", "windowedLogRealTimeAlertCalculator", "timeService", + "windowAggregator", "grafanaInit", "sopScheduleExecutor", "greptimeApmFlowInitializer", + "greptimeLogPipelineInitializer", "greptimeTraceTableInitializer", + "greptimeTtlApplicationReadyListener", "calculateStatusLifecycle", "alarmReduceLifecycle", + "legacyLogSseManagerLifecycle", "observabilityLogSseManagerLifecycle", + "managerBusinessRuntimeInitializer", "llmConfigInitializer", "dorisDataStorage", + "tdEngineDataStorage", "influxdbDataStorage", "iotDbDataStorage", "duckdbDatabaseDataStorage", + "greptimeDbDataStorage", "victoriaMetricsDataStorage", "victoriaMetricsClusterDataStorage", + "questdbDataStorage" + }; + + @Test + void realFullApplicationStartsGatedWithoutBusinessSideEffectsOrCliBypass() { + SpringStartupContextLauncher launcher = new SpringStartupContextLauncher(); + StartupDecision decision = new StartupDecision( + RuntimeMode.FULL_SETUP_GATED, SetupPhase.ADMINISTRATOR_REQUIRED, null); + String databaseName = "m2_gated_" + System.nanoTime(); + try (ConfigurableApplicationContext context = launcher.launchSpringContext(decision, new String[]{ + "--spring.profiles.active=test", + "--spring.main.web-application-type=none", + "--spring.datasource.url=jdbc:h2:mem:" + databaseName + ";MODE=MYSQL;DB_CLOSE_DELAY=-1", + "--spring.flyway.enabled=false", + "--warehouse.store.doris.enabled=true", + "--warehouse.store.td-engine.enabled=true", + "--warehouse.store.influxdb.enabled=true", + "--warehouse.store.iot-db.enabled=true", + "--warehouse.store.duckdb.enabled=true", + "--warehouse.store.greptime.enabled=true", + "--warehouse.store.victoria-metrics.enabled=true", + "--warehouse.store.victoria-metrics.cluster.enabled=true", + "--warehouse.store.questdb.enabled=true", + "--hertzbeat.runtime.mode=normal" + }, SETUP_RUNTIME_TRANSITION)) { + BusinessRuntimeGate gate = context.getBean(BusinessRuntimeGate.class); + assertSame(SETUP_RUNTIME_TRANSITION, context.getBean(SetupRuntimeTransition.class)); + assertEquals(RuntimeMode.FULL_SETUP_GATED, gate.mode()); + assertFalse(gate.isOpen()); + assertTrue(context.containsBeanDefinition("periodicAlertRuleScheduler")); + assertTrue(context.containsBeanDefinition("manageServer")); + assertTrue(context.containsBeanDefinition("otlpGrpcMetricsService")); + assertTrue(context.containsBeanDefinition("alarmGroupReduce")); + assertTrue(context.containsBeanDefinition("alarmInhibitReduce")); + assertTrue(context.containsBeanDefinition("calculateStatus")); + assertTrue(context.containsBeanDefinition("objectStoreConfigServiceImpl")); + assertTrue(context.containsBeanDefinition("appServiceImpl")); + assertTrue(context.containsBeanDefinition("pluginParameterServiceImpl")); + assertTrue(context.containsBeanDefinition("pluginServiceImpl")); + assertTrue(context.containsBeanDefinition("llmConfig")); + for (String beanName : SIDE_EFFECT_BEANS) { + assertFalse(context.containsBeanDefinition(beanName), beanName); + } + } + } + + @Test + void setupOnlySourceStartsWithoutBusinessAutoConfiguration() { + SpringStartupContextLauncher launcher = new SpringStartupContextLauncher(); + StartupDecision decision = new StartupDecision( + RuntimeMode.SETUP_ONLY, SetupPhase.CONFIGURATION_REQUIRED, null); + try (ConfigurableApplicationContext context = launcher.launchSpringContext(decision, + new String[]{"--spring.main.web-application-type=none", "--hertzbeat.runtime.mode=normal"}, + SETUP_RUNTIME_TRANSITION)) { + BusinessRuntimeGate gate = context.getBean(BusinessRuntimeGate.class); + assertSame(SETUP_RUNTIME_TRANSITION, context.getBean(SetupRuntimeTransition.class)); + assertEquals(RuntimeMode.SETUP_ONLY, gate.mode()); + assertFalse(context.containsBeanDefinition("managerAutoConfiguration")); + assertFalse(context.containsBeanDefinition("entityManagerFactory")); + assertFalse(context.containsBeanDefinition("dataSource")); + assertFalse(context.containsBeanDefinition("flyway")); + assertFalse(context.containsBeanDefinition("grafanaAutoConfiguration")); + assertFalse(context.containsBeanDefinition("alerterAutoConfiguration")); + assertFalse(context.containsBeanDefinition("collectorAutoConfiguration")); + assertFalse(context.containsBeanDefinition("warehouseAutoConfiguration")); + assertFalse(context.containsBeanDefinition("otlpGrpcServerConfig")); + } + } +} diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/listener/GreptimeTtlApplicationReadyListener.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/listener/GreptimeTtlApplicationReadyListener.java index dd740d8757..358f3899b6 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/listener/GreptimeTtlApplicationReadyListener.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/listener/GreptimeTtlApplicationReadyListener.java @@ -21,6 +21,7 @@ import java.time.temporal.TemporalAmount; import java.util.regex.Pattern; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.math.NumberUtils; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.apache.hertzbeat.common.util.TimePeriodUtil; import org.apache.hertzbeat.warehouse.db.GreptimeSqlQueryExecutor; import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties; @@ -35,6 +36,7 @@ import org.springframework.util.StringUtils; */ @Slf4j @Component +@ConditionalOnNormalBusinessRuntime @ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true") public class GreptimeTtlApplicationReadyListener { diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/doris/DorisDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/doris/DorisDataStorage.java index 6c9b0eb2ff..29f3d9d94c 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/doris/DorisDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/doris/DorisDataStorage.java @@ -27,6 +27,7 @@ import org.apache.hertzbeat.common.entity.arrow.RowWrapper; import org.apache.hertzbeat.common.entity.dto.Value; import org.apache.hertzbeat.common.entity.log.LogEntry; import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.common.util.TimePeriodUtil; import org.apache.hertzbeat.warehouse.WarehouseWorkerPool; @@ -71,6 +72,7 @@ import java.util.concurrent.atomic.AtomicBoolean; * - stream: HTTP Stream Load API (high throughput, suitable for large scale) */ @Component +@ConditionalOnNormalBusinessRuntime @ConditionalOnProperty(prefix = "warehouse.store.doris", name = "enabled", havingValue = "true") @Slf4j public class DorisDataStorage extends AbstractHistoryDataStorage { diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorage.java index e7481406d7..39b5ed3485 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorage.java @@ -27,6 +27,7 @@ import org.apache.hertzbeat.common.constants.MetricDataConstants; import org.apache.hertzbeat.common.entity.arrow.RowWrapper; import org.apache.hertzbeat.common.entity.dto.Value; import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.common.util.TimePeriodUtil; import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage; @@ -60,6 +61,7 @@ import java.util.regex.Pattern; * data storage by duckdb */ @Component +@ConditionalOnNormalBusinessRuntime @ConditionalOnProperty(prefix = "warehouse.store.duckdb", name = "enabled", havingValue = "true") @Slf4j public class DuckdbDatabaseDataStorage extends AbstractHistoryDataStorage { diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorage.java index c40a17604b..5acd4a9e17 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorage.java @@ -66,6 +66,7 @@ import org.apache.hertzbeat.common.entity.arrow.RowWrapper; import org.apache.hertzbeat.common.entity.dto.Value; import org.apache.hertzbeat.common.entity.log.LogEntry; import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.apache.hertzbeat.common.util.Base64Util; import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.common.util.TimePeriodUtil; @@ -92,6 +93,7 @@ import org.springframework.web.util.UriComponentsBuilder; * GreptimeDB data storage, only supports GreptimeDB version >= v0.5 */ @Component +@ConditionalOnNormalBusinessRuntime @ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true") @Slf4j public class GreptimeDbDataStorage extends AbstractHistoryDataStorage { diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/influxdb/InfluxdbDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/influxdb/InfluxdbDataStorage.java index f65d6673aa..046b5025ab 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/influxdb/InfluxdbDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/influxdb/InfluxdbDataStorage.java @@ -45,6 +45,7 @@ import org.apache.hertzbeat.common.constants.NetworkConstants; import org.apache.hertzbeat.common.entity.arrow.RowWrapper; import org.apache.hertzbeat.common.entity.dto.Value; import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage; import org.apache.http.ssl.SSLContexts; @@ -61,6 +62,7 @@ import org.springframework.stereotype.Component; * HistoryInfluxdbDataStorage class */ @Component +@ConditionalOnNormalBusinessRuntime @ConditionalOnProperty(prefix = "warehouse.store.influxdb", name = "enabled", havingValue = "true") @Slf4j public class InfluxdbDataStorage extends AbstractHistoryDataStorage { diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/iotdb/IotDbDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/iotdb/IotDbDataStorage.java index 0b79ce3e9b..9eb591297c 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/iotdb/IotDbDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/iotdb/IotDbDataStorage.java @@ -32,6 +32,7 @@ import org.apache.hertzbeat.common.constants.MetricDataConstants; import org.apache.hertzbeat.common.entity.arrow.RowWrapper; import org.apache.hertzbeat.common.entity.dto.Value; import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage; import org.apache.iotdb.rpc.IoTDBConnectionException; @@ -49,6 +50,7 @@ import org.springframework.stereotype.Component; * IoTDB data storage */ @Component +@ConditionalOnNormalBusinessRuntime @ConditionalOnProperty(prefix = "warehouse.store.iot-db", name = "enabled", havingValue = "true") @Slf4j public class IotDbDataStorage extends AbstractHistoryDataStorage { diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/questdb/QuestdbDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/questdb/QuestdbDataStorage.java index 801d635295..96b1c53c10 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/questdb/QuestdbDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/questdb/QuestdbDataStorage.java @@ -51,6 +51,7 @@ import org.apache.hertzbeat.common.constants.NetworkConstants; import org.apache.hertzbeat.common.entity.arrow.RowWrapper; import org.apache.hertzbeat.common.entity.dto.Value; import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage; import org.apache.http.ssl.SSLContexts; @@ -61,6 +62,7 @@ import org.springframework.stereotype.Component; * HistoryQuestdbDataStorage class */ @Component +@ConditionalOnNormalBusinessRuntime @ConditionalOnProperty(prefix = "warehouse.store.questdb", name = "enabled", havingValue = "true") @Slf4j public class QuestdbDataStorage extends AbstractHistoryDataStorage { @@ -411,4 +413,4 @@ public class QuestdbDataStorage extends AbstractHistoryDataStorage { this.client.dispatcher().executorService().shutdown(); } } -} \ No newline at end of file +} diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/tdengine/TdEngineDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/tdengine/TdEngineDataStorage.java index 308e15b0bc..4c086e6996 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/tdengine/TdEngineDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/tdengine/TdEngineDataStorage.java @@ -46,6 +46,7 @@ import org.apache.hertzbeat.common.constants.MetricDataConstants; import org.apache.hertzbeat.common.entity.arrow.RowWrapper; import org.apache.hertzbeat.common.entity.dto.Value; import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.common.util.StrBuffer; import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage; @@ -59,6 +60,7 @@ import org.springframework.stereotype.Component; */ @Primary @Component +@ConditionalOnNormalBusinessRuntime @ConditionalOnProperty(prefix = "warehouse.store.td-engine", name = "enabled", havingValue = "true") @Slf4j diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/vm/VictoriaMetricsClusterDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/vm/VictoriaMetricsClusterDataStorage.java index e8b6410a46..af66012dd0 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/vm/VictoriaMetricsClusterDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/vm/VictoriaMetricsClusterDataStorage.java @@ -53,6 +53,7 @@ import org.apache.hertzbeat.common.constants.SignConstants; import org.apache.hertzbeat.common.entity.arrow.RowWrapper; import org.apache.hertzbeat.common.entity.dto.Value; import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.apache.hertzbeat.common.timer.HashedWheelTimer; import org.apache.hertzbeat.common.timer.Timeout; import org.apache.hertzbeat.common.timer.TimerTask; @@ -81,6 +82,7 @@ import static org.apache.hertzbeat.common.constants.ConfigConstants.FunctionModu */ @Primary @Component +@ConditionalOnNormalBusinessRuntime @ConditionalOnProperty(prefix = "warehouse.store.victoria-metrics.cluster", name = "enabled", havingValue = "true") @Slf4j public class VictoriaMetricsClusterDataStorage extends AbstractHistoryDataStorage { diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/vm/VictoriaMetricsDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/vm/VictoriaMetricsDataStorage.java index 5eb826811a..3636f30edb 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/vm/VictoriaMetricsDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/vm/VictoriaMetricsDataStorage.java @@ -52,6 +52,7 @@ import org.apache.hertzbeat.common.constants.SignConstants; import org.apache.hertzbeat.common.entity.arrow.RowWrapper; import org.apache.hertzbeat.common.entity.dto.Value; import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.apache.hertzbeat.common.timer.HashedWheelTimer; import org.apache.hertzbeat.common.timer.Timeout; import org.apache.hertzbeat.common.timer.TimerTask; @@ -78,6 +79,7 @@ import org.springframework.web.util.UriComponentsBuilder; */ @Primary @Component +@ConditionalOnNormalBusinessRuntime @ConditionalOnProperty(prefix = "warehouse.store.victoria-metrics", name = "enabled", havingValue = "true") @Slf4j public class VictoriaMetricsDataStorage extends AbstractHistoryDataStorage { From 5a399175cb9c787d6d76867971c9bb02980a34bd Mon Sep 17 00:00:00 2001 From: Logic Date: Sat, 8 Aug 2026 07:59:12 +0800 Subject: [PATCH 09/71] Add managed deployment configuration --- .../manager/setup/api/SetupApiContract.java | 17 +- .../ApplicationConfigDocumentCodec.java | 131 ++++++++ .../manager/setup/config/CandidateRead.java | 60 ++++ .../manager/setup/config/CandidateState.java | 27 ++ .../setup/config/DeploymentConstraint.java | 28 ++ .../EffectiveConfigurationResolver.java | 89 +++++ .../config/EffectiveConfigurationValue.java | 40 +++ .../config/ExternalConfigExportArtifact.java | 53 +++ .../FileManagedApplicationConfigStore.java | 79 +++++ .../setup/config/FileManagedSecretStore.java | 78 +++++ .../config/FileManagedSnapshotStore.java | 146 ++++++++ .../setup/config/GreptimeEndpoints.java | 38 +++ .../setup/config/GreptimeSettings.java | 50 +++ .../ManagedActiveConfigurationInspector.java | 112 +++++++ .../config/ManagedApplicationConfig.java | 36 ++ .../config/ManagedApplicationConfigStore.java | 38 +++ .../setup/config/ManagedConfigCapability.java | 45 +++ .../ManagedConfigDeploymentDetector.java | 108 ++++++ .../config/ManagedConfigurationBundle.java | 39 +++ .../ManagedConfigurationTransaction.java | 304 +++++++++++++++++ .../setup/config/ManagedDocumentCodec.java | 141 ++++++++ .../manager/setup/config/ManagedFileIo.java | 48 +++ .../setup/config/ManagedSecretStore.java | 38 +++ .../manager/setup/config/ManagedSecrets.java | 44 +++ .../config/MetadataDatabaseSettings.java | 42 +++ .../setup/config/NioManagedFilePublisher.java | 105 ++++++ .../setup/config/RestartRequirement.java | 24 ++ .../config/SecretConfigDocumentCodec.java | 133 ++++++++ .../manager/setup/config/SecretValue.java | 56 ++++ .../setup/config/SensitiveExportContent.java | 44 +++ .../setup/api/SetupApiContractTest.java | 17 + .../EffectiveConfigurationResolverTest.java | 56 ++++ .../ExternalConfigExportArtifactTest.java | 65 ++++ .../FileManagedConfigurationStoreTest.java | 248 ++++++++++++++ .../ManagedConfigurationPortContractTest.java | 161 +++++++++ .../ManagedConfigurationTransactionTest.java | 316 ++++++++++++++++++ .../ManagedDeploymentCapabilityTest.java | 164 +++++++++ .../config/NioManagedFilePublisherTest.java | 87 +++++ hertzbeat-startup/pom.xml | 1 + ...ManagedConfigEnvironmentPostProcessor.java | 124 +++++++ .../runtime/SpringStartupContextLauncher.java | 10 +- .../main/resources/META-INF/spring.factories | 2 + .../ManagedConfigDataPrecedenceTest.java | 166 +++++++++ .../ManagedConfigRecoveryContextTest.java | 140 ++++++++ 44 files changed, 3745 insertions(+), 5 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ApplicationConfigDocumentCodec.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/CandidateRead.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/CandidateState.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/DeploymentConstraint.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/EffectiveConfigurationResolver.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/EffectiveConfigurationValue.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ExternalConfigExportArtifact.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedApplicationConfigStore.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedSecretStore.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedSnapshotStore.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/GreptimeEndpoints.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/GreptimeSettings.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedActiveConfigurationInspector.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedApplicationConfig.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedApplicationConfigStore.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigCapability.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigDeploymentDetector.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationBundle.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransaction.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedDocumentCodec.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedFileIo.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedSecretStore.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedSecrets.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MetadataDatabaseSettings.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/NioManagedFilePublisher.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/RestartRequirement.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SecretConfigDocumentCodec.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SecretValue.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SensitiveExportContent.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/EffectiveConfigurationResolverTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ExternalConfigExportArtifactTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/FileManagedConfigurationStoreTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationPortContractTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedDeploymentCapabilityTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/NioManagedFilePublisherTest.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/config/ManagedConfigEnvironmentPostProcessor.java create mode 100644 hertzbeat-startup/src/main/resources/META-INF/spring.factories create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/config/ManagedConfigDataPrecedenceTest.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/ManagedConfigRecoveryContextTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java index 073e86abbd..decdd2c8f0 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java @@ -372,13 +372,22 @@ public final class SetupApiContract { @NotBlank String grpcEndpoints, @NotBlank String httpEndpoint, @NotBlank String database, - @NotBlank String username, - @NotBlank @JsonProperty(access = JsonProperty.Access.WRITE_ONLY) String password) { + String username, + @JsonProperty(access = JsonProperty.Access.WRITE_ONLY) String password) { public TelemetryStoreConfiguration { if (kind != TelemetryStoreKind.GREPTIME) { throw new IllegalArgumentException("Only Greptime telemetry storage is supported"); } + username = normalizeCredential(username); + password = normalizeCredential(password); + if (hasText(username) != hasText(password)) { + throw new IllegalArgumentException("Greptime username and password must be supplied together"); + } + } + + private static String normalizeCredential(String value) { + return value == null || value.isBlank() ? null : value; } @Override @@ -593,4 +602,8 @@ public final class SetupApiContract { throw new IllegalArgumentException("Retention days must be positive when supplied"); } } + + private static boolean hasText(String value) { + return value != null && !value.isBlank(); + } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ApplicationConfigDocumentCodec.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ApplicationConfigDocumentCodec.java new file mode 100644 index 0000000000..6b2028d4f1 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ApplicationConfigDocumentCodec.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.yaml.snakeyaml.LoaderOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.SafeConstructor; +import org.yaml.snakeyaml.error.YAMLException; + +final class ApplicationConfigDocumentCodec implements ManagedDocumentCodec { + + // This exact flat-key allowlist is the boundary that prevents setup from becoming an arbitrary YAML editor. + private static final String DATASOURCE_URL = "spring.datasource.url"; + private static final String DATASOURCE_USERNAME = "spring.datasource.username"; + private static final String DATABASE_KIND = "spring.jpa.database"; + private static final String DUCKDB_ENABLED = "warehouse.store.duckdb.enabled"; + private static final String GREPTIME_ENABLED = "warehouse.store.greptime.enabled"; + private static final String GREPTIME_GRPC = "warehouse.store.greptime.grpc-endpoints"; + private static final String GREPTIME_HTTP = "warehouse.store.greptime.http-endpoint"; + private static final String GREPTIME_DATABASE = "warehouse.store.greptime.database"; + private static final String GREPTIME_USERNAME = "warehouse.store.greptime.username"; + private static final Set REQUIRED_KEYS = Set.of( + DATASOURCE_URL, DATASOURCE_USERNAME, DATABASE_KIND, + DUCKDB_ENABLED, GREPTIME_ENABLED, + GREPTIME_GRPC, GREPTIME_HTTP, GREPTIME_DATABASE); + + @Override + public byte[] encode(ManagedApplicationConfig value, String generation) { + Map values = plainProperties(value); + StringBuilder body = new StringBuilder(); + values.forEach((key, item) -> body.append(key).append(": '") + .append(item.replace("'", "''")).append("'\n")); + return Integrity.envelope(body.toString(), generation); + } + + static Map springProperties(ManagedApplicationConfig value) { + Map properties = new LinkedHashMap<>(); + plainProperties(value).forEach( + (key, item) -> properties.put(key, Integrity.literalForSpring(item))); + return Map.copyOf(properties); + } + + private static Map plainProperties(ManagedApplicationConfig value) { + Map values = new LinkedHashMap<>(); + values.put(DATASOURCE_URL, value.metadataDatabase().jdbcUrl()); + values.put(DATASOURCE_USERNAME, value.metadataDatabase().username()); + values.put(DATABASE_KIND, value.metadataDatabase().kind().name()); + values.put(DUCKDB_ENABLED, "false"); + values.put(GREPTIME_ENABLED, "true"); + values.put(GREPTIME_GRPC, value.telemetryStore().endpoints().grpc()); + values.put(GREPTIME_HTTP, value.telemetryStore().endpoints().http()); + values.put(GREPTIME_DATABASE, value.telemetryStore().database()); + value.telemetryStore().username().ifPresent(username -> values.put(GREPTIME_USERNAME, username)); + return values; + } + + @Override + public Decoded decode(byte[] content) + throws DocumentException { + Integrity.VerifiedBody body = Integrity.extract(content); + Integrity.verify(body); + Map values; + try { + Object loaded = new Yaml(new SafeConstructor(new LoaderOptions())).load(body.content()); + if (!(loaded instanceof Map loadedMap)) { + throw DocumentException.corrupt(); + } + values = loadedMap; + } catch (YAMLException exception) { + throw DocumentException.corrupt(); + } + if (!values.keySet().stream().allMatch(String.class::isInstance) + || !values.keySet().containsAll(REQUIRED_KEYS) + || values.size() > REQUIRED_KEYS.size() + 1 + || (values.size() > REQUIRED_KEYS.size() && !values.containsKey(GREPTIME_USERNAME)) + || !usesSupportedTelemetryStorage(values)) { + throw DocumentException.corrupt(); + } + ManagedApplicationConfig decoded; + try { + MetadataDatabaseSettings metadata = new MetadataDatabaseSettings( + MetadataDatabaseKind.valueOf(text(values, DATABASE_KIND)), + text(values, DATASOURCE_URL), text(values, DATASOURCE_USERNAME)); + GreptimeEndpoints endpoints = new GreptimeEndpoints( + text(values, GREPTIME_GRPC), text(values, GREPTIME_HTTP)); + decoded = new ManagedApplicationConfig(metadata, values.containsKey(GREPTIME_USERNAME) + ? GreptimeSettings.authenticated( + endpoints, text(values, GREPTIME_DATABASE), text(values, GREPTIME_USERNAME)) + : GreptimeSettings.anonymous(endpoints, text(values, GREPTIME_DATABASE))); + } catch (IllegalArgumentException exception) { + throw DocumentException.invalid(); + } + return new Decoded<>(decoded, body.generation()); + } + + /** + * Managed setup configuration has one supported telemetry-storage policy. Checking the values as + * well as the keys prevents a hand-edited document from silently enabling an unsupported store. + */ + private static boolean usesSupportedTelemetryStorage(Map values) { + return "false".equals(values.get(DUCKDB_ENABLED)) + && "true".equals(values.get(GREPTIME_ENABLED)); + } + + private static String text(Map values, String key) { + Object value = values.get(key); + if (!(value instanceof String text)) { + throw new IllegalArgumentException("Missing managed value"); + } + return text; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/CandidateRead.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/CandidateRead.java new file mode 100644 index 0000000000..f3575aed14 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/CandidateRead.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.util.Objects; +import java.util.Optional; + +/** Secret-free result of reading a candidate or last-known-good snapshot. */ +record CandidateRead(CandidateState state, Optional value, Optional generation) { + + public CandidateRead { + Objects.requireNonNull(state, "state"); + Objects.requireNonNull(value, "value"); + Objects.requireNonNull(generation, "generation"); + if ((state == CandidateState.VALID) != value.isPresent() + || value.isPresent() != generation.isPresent()) { + throw new IllegalArgumentException("Only a valid read may contain a value and generation"); + } + generation.ifPresent(ManagedDocumentCodec.Integrity::requireValidGeneration); + } + + public static CandidateRead valid(T value, String generation) { + return new CandidateRead<>(CandidateState.VALID, Optional.of(value), Optional.of(generation)); + } + + public static CandidateRead missing() { + return nonValid(CandidateState.MISSING); + } + + public static CandidateRead invalid() { + return nonValid(CandidateState.INVALID); + } + + public static CandidateRead unreadable() { + return nonValid(CandidateState.UNREADABLE); + } + + public static CandidateRead corrupt() { + return nonValid(CandidateState.CORRUPT); + } + + private static CandidateRead nonValid(CandidateState state) { + return new CandidateRead<>(state, Optional.empty(), Optional.empty()); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/CandidateState.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/CandidateState.java new file mode 100644 index 0000000000..1c0880ffb6 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/CandidateState.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +/** Safe recovery classification for one persisted snapshot. */ +enum CandidateState { + MISSING, + VALID, + INVALID, + UNREADABLE, + CORRUPT +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/DeploymentConstraint.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/DeploymentConstraint.java new file mode 100644 index 0000000000..24ce55efdf --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/DeploymentConstraint.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +/** Safe reason that managed setup cannot write its configuration. */ +public enum DeploymentConstraint { + NONE, + READ_ONLY, + INSTALLATION_ROOT_MISSING, + INSTALLATION_ROOT_NOT_DIRECTORY, + CONFIG_PATH_NOT_DIRECTORY, + UNSAFE_PATH +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/EffectiveConfigurationResolver.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/EffectiveConfigurationResolver.java new file mode 100644 index 0000000000..3591919a87 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/EffectiveConfigurationResolver.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigSource; +import org.springframework.boot.origin.Origin; +import org.springframework.boot.origin.OriginLookup; +import org.springframework.boot.origin.OriginTrackedResource; +import org.springframework.boot.origin.TextResourceOrigin; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.core.env.Environment; +import org.springframework.core.env.PropertySource; +import org.springframework.core.env.StandardEnvironment; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; + +/** Resolves the supported configuration layers using Spring's documented precedence. */ +public final class EffectiveConfigurationResolver { + + private static final String CONFIGURATION_PROPERTIES = "configurationProperties"; + + public EffectiveConfigurationValue resolve( + Environment environment, String key, RestartRequirement restartRequirement) { + Objects.requireNonNull(environment, "environment"); + Objects.requireNonNull(key, "key"); + if (!(environment instanceof ConfigurableEnvironment configurable)) { + throw new IllegalArgumentException("A configurable Spring environment is required"); + } + String value = environment.getProperty(key); + if (value == null) { + throw new IllegalArgumentException("Configuration key is unavailable: " + key); + } + for (PropertySource propertySource : configurable.getPropertySources()) { + if (!CONFIGURATION_PROPERTIES.equals(propertySource.getName()) + && propertySource.getProperty(key) != null) { + return new EffectiveConfigurationValue<>( + value, classify(propertySource, key), restartRequirement); + } + } + throw new IllegalStateException("Configuration source is unavailable for key: " + key); + } + + private static ConfigSource classify(PropertySource propertySource, String key) { + String sourceName = propertySource.getName(); + if ("commandLineArgs".equals(sourceName)) { + return ConfigSource.COMMAND_LINE; + } + if (StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME.equals(sourceName)) { + return ConfigSource.SYSTEM_PROPERTY; + } + if (StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME.equals(sourceName)) { + return ConfigSource.ENVIRONMENT; + } + if (ManagedActiveConfigurationInspector.MANAGED_APPLICATION_SOURCE.equals(sourceName) + || ManagedActiveConfigurationInspector.MANAGED_SECRET_SOURCE.equals(sourceName)) { + return ConfigSource.UI_MANAGED; + } + Origin origin = OriginLookup.getOrigin(propertySource, key); + if (origin instanceof TextResourceOrigin textOrigin) { + return unwrap(textOrigin.getResource()) instanceof ClassPathResource + ? ConfigSource.BUILT_IN_DEFAULT : ConfigSource.EXTERNAL_FILE; + } + throw new IllegalStateException("Unsupported configuration property source: " + sourceName); + } + + private static Resource unwrap(Resource resource) { + Resource current = resource; + while (current instanceof OriginTrackedResource tracked) { + current = tracked.getResource(); + } + return current; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/EffectiveConfigurationValue.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/EffectiveConfigurationValue.java new file mode 100644 index 0000000000..8bfc674f79 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/EffectiveConfigurationValue.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigSource; + +/** Effective typed value together with its source and activation behavior. */ +public record EffectiveConfigurationValue( + T value, + ConfigSource source, + RestartRequirement restartRequirement) { + + public EffectiveConfigurationValue { + Objects.requireNonNull(value, "value"); + Objects.requireNonNull(source, "source"); + Objects.requireNonNull(restartRequirement, "restartRequirement"); + } + + @Override + public String toString() { + return "EffectiveConfigurationValue[value=, source=" + source + + ", restartRequirement=" + restartRequirement + "]"; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ExternalConfigExportArtifact.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ExternalConfigExportArtifact.java new file mode 100644 index 0000000000..e2267e71e1 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ExternalConfigExportArtifact.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; + +/** Safe attachment metadata and sensitive bytes for operator-applied configuration. */ +public record ExternalConfigExportArtifact( + String fileName, + String mediaType, + SensitiveExportContent content) { + + public ExternalConfigExportArtifact { + if (fileName == null || !fileName.matches("[A-Za-z0-9._-]+")) { + throw new IllegalArgumentException("Export filename is unsafe"); + } + if (mediaType == null || mediaType.isBlank() || mediaType.indexOf('\\') >= 0 + || mediaType.indexOf('\r') >= 0 || mediaType.indexOf('\n') >= 0) { + throw new IllegalArgumentException("Export media type is unsafe"); + } + Objects.requireNonNull(content, "content"); + } + + public SetupOperationState state() { + return SetupOperationState.AWAITING_EXTERNAL_APPLY; + } + + public boolean noStore() { + return true; + } + + @Override + public String toString() { + return "ExternalConfigExportArtifact[fileName=" + fileName + ", mediaType=" + mediaType + + ", content=, state=awaiting_external_apply, noStore=true]"; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedApplicationConfigStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedApplicationConfigStore.java new file mode 100644 index 0000000000..4df48d6a47 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedApplicationConfigStore.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +/** File adapter owning only the managed non-secret application overlay transaction. */ +final class FileManagedApplicationConfigStore implements ManagedApplicationConfigStore { + + private static final String FILE_NAME = "managed-application.yml"; + + private final FileManagedSnapshotStore delegate; + + FileManagedApplicationConfigStore(Path installationRoot) { + this(installationRoot, new NioManagedFilePublisher(), Files::readAllBytes); + } + + FileManagedApplicationConfigStore( + Path installationRoot, ManagedFileIo.Publisher publisher, ManagedFileIo.Reader reader) { + delegate = new FileManagedSnapshotStore<>(installationRoot, FILE_NAME, false, + new ApplicationConfigDocumentCodec(), publisher, reader); + } + + FileManagedApplicationConfigStore(Path installationRoot, ManagedFileIo.Publisher publisher) { + this(installationRoot, publisher, Files::readAllBytes); + } + + @Override + public CandidateRead readCandidate() { + return delegate.readCandidate(); + } + + @Override + public CandidateRead readActive() { + return delegate.readActive(); + } + + @Override + public CandidateRead readLastKnownGood() { + return delegate.readLastKnownGood(); + } + + @Override + public void stageCandidate(ManagedApplicationConfig candidate, String generation) throws IOException { + delegate.stageCandidate(candidate, generation); + } + + @Override + public void promoteCandidate(ManagedApplicationConfig expected, String generation) throws IOException { + delegate.promoteCandidate(expected, generation); + } + + @Override + public void restoreActive(CandidateRead previous) throws IOException { + delegate.restoreActive(previous.value(), previous.generation()); + } + + @Override + public void discardCandidate() throws IOException { + delegate.discardCandidate(); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedSecretStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedSecretStore.java new file mode 100644 index 0000000000..4a1cc043c1 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedSecretStore.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +/** File adapter owning only the managed secret properties transaction. */ +final class FileManagedSecretStore implements ManagedSecretStore { + + private static final String FILE_NAME = "managed-secrets.properties"; + + private final FileManagedSnapshotStore delegate; + + FileManagedSecretStore(Path installationRoot) { + this(installationRoot, new NioManagedFilePublisher(), Files::readAllBytes); + } + + FileManagedSecretStore(Path installationRoot, ManagedFileIo.Publisher publisher, ManagedFileIo.Reader reader) { + delegate = new FileManagedSnapshotStore<>(installationRoot, FILE_NAME, true, + new SecretConfigDocumentCodec(), publisher, reader); + } + + FileManagedSecretStore(Path installationRoot, ManagedFileIo.Publisher publisher) { + this(installationRoot, publisher, Files::readAllBytes); + } + + @Override + public CandidateRead readCandidate() { + return delegate.readCandidate(); + } + + @Override + public CandidateRead readActive() { + return delegate.readActive(); + } + + @Override + public CandidateRead readLastKnownGood() { + return delegate.readLastKnownGood(); + } + + @Override + public void stageCandidate(ManagedSecrets candidate, String generation) throws IOException { + delegate.stageCandidate(candidate, generation); + } + + @Override + public void promoteCandidate(ManagedSecrets expected, String generation) throws IOException { + delegate.promoteCandidate(expected, generation); + } + + @Override + public void restoreActive(CandidateRead previous) throws IOException { + delegate.restoreActive(previous.value(), previous.generation()); + } + + @Override + public void discardCandidate() throws IOException { + delegate.discardCandidate(); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedSnapshotStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedSnapshotStore.java new file mode 100644 index 0000000000..f257b66afa --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedSnapshotStore.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.util.Optional; + +final class FileManagedSnapshotStore { + + private static final String CANDIDATE_SUFFIX = ".candidate"; + private static final String LAST_KNOWN_GOOD_SUFFIX = ".last-known-good"; + + private final Path active; + private final Path candidate; + private final Path lastKnownGood; + private final boolean ownerOnly; + private final ManagedDocumentCodec codec; + private final ManagedFileIo.Publisher publisher; + private final ManagedFileIo.Reader reader; + private final Path installationRoot; + + FileManagedSnapshotStore( + Path installationRoot, + String fileName, + boolean ownerOnly, + ManagedDocumentCodec codec, + ManagedFileIo.Publisher publisher, + ManagedFileIo.Reader reader) { + this.installationRoot = installationRoot.toAbsolutePath().normalize(); + Path configDirectory = this.installationRoot.resolve("data/config"); + this.active = configDirectory.resolve(fileName); + this.candidate = active.resolveSibling(fileName + CANDIDATE_SUFFIX); + this.lastKnownGood = active.resolveSibling(fileName + LAST_KNOWN_GOOD_SUFFIX); + this.ownerOnly = ownerOnly; + this.codec = codec; + this.publisher = publisher; + this.reader = reader; + } + + CandidateRead readCandidate() { + return read(candidate); + } + + CandidateRead readActive() { + return read(active); + } + + CandidateRead readLastKnownGood() { + return read(lastKnownGood); + } + + void stageCandidate(T value, String generation) throws IOException { + ensureSafePaths(); + publisher.publish(candidate, codec.encode(value, generation), ownerOnly); + } + + void promoteCandidate(T expected, String generation) throws IOException { + ensureSafePaths(); + byte[] candidateDocument; + ManagedDocumentCodec.Decoded decoded; + try { + candidateDocument = reader.read(candidate); + decoded = codec.decode(candidateDocument); + } catch (ManagedDocumentCodec.DocumentException | IOException failure) { + throw new IOException("A valid managed configuration candidate is required"); + } + if (!expected.equals(decoded.value()) || !generation.equals(decoded.generation())) { + throw new IOException("Managed configuration candidate does not match the transaction"); + } + CandidateRead activeRead = readActive(); + if (activeRead.state() == CandidateState.VALID) { + publisher.publish(lastKnownGood, codec.encode( + activeRead.value().orElseThrow(), activeRead.generation().orElseThrow()), ownerOnly); + } else if (activeRead.state() != CandidateState.MISSING) { + throw new IOException("Active managed configuration requires recovery"); + } + publisher.publish(active, candidateDocument, ownerOnly); + publisher.remove(candidate); + } + + void restoreActive(Optional previous, Optional generation) throws IOException { + ensureSafePaths(); + if (previous.isPresent()) { + byte[] document = codec.encode(previous.orElseThrow(), generation.orElseThrow()); + publisher.publish(active, document, ownerOnly); + publisher.publish(lastKnownGood, document, ownerOnly); + } else { + publisher.remove(active); + publisher.remove(lastKnownGood); + } + } + + void discardCandidate() throws IOException { + ensureSafePaths(); + publisher.remove(candidate); + } + + private CandidateRead read(Path path) { + if (isUnsafePath(path)) { + return CandidateRead.unreadable(); + } + try { + ManagedDocumentCodec.Decoded decoded = codec.decode(reader.read(path)); + return CandidateRead.valid(decoded.value(), decoded.generation()); + } catch (ManagedDocumentCodec.DocumentException exception) { + return exception.state() == CandidateState.INVALID ? CandidateRead.invalid() : CandidateRead.corrupt(); + } catch (NoSuchFileException exception) { + return CandidateRead.missing(); + } catch (IOException exception) { + return CandidateRead.unreadable(); + } + } + + private void ensureSafePaths() throws IOException { + Path data = installationRoot.resolve("data"); + Path config = data.resolve("config"); + if (isUnsafePath(installationRoot) || isUnsafePath(data) || isUnsafePath(config) || isUnsafePath(active) + || isUnsafePath(candidate) || isUnsafePath(lastKnownGood) + || (Files.exists(data) && !Files.isDirectory(data)) + || (Files.exists(config) && !Files.isDirectory(config))) { + throw new IOException("Managed configuration path is unsafe"); + } + } + + private static boolean isUnsafePath(Path path) { + return Files.isSymbolicLink(path); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/GreptimeEndpoints.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/GreptimeEndpoints.java new file mode 100644 index 0000000000..214e335b25 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/GreptimeEndpoints.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +/** GreptimeDB ingestion and query endpoints. */ +public record GreptimeEndpoints(String grpc, String http) { + + public GreptimeEndpoints { + requireText(grpc, "grpc"); + requireText(http, "http"); + } + + @Override + public String toString() { + return "GreptimeEndpoints[configured=true]"; + } + + private static void requireText(String value, String name) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException(name + " must not be blank"); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/GreptimeSettings.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/GreptimeSettings.java new file mode 100644 index 0000000000..25c2c054cf --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/GreptimeSettings.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.util.Objects; +import java.util.Optional; + +/** Supported non-secret GreptimeDB settings, including anonymous local deployments. */ +public record GreptimeSettings(GreptimeEndpoints endpoints, String database, Optional username) { + + public GreptimeSettings { + Objects.requireNonNull(endpoints, "endpoints"); + if (database == null || database.isBlank()) { + throw new IllegalArgumentException("database must not be blank"); + } + Objects.requireNonNull(username, "username"); + username = username.map(String::trim).filter(value -> !value.isEmpty()); + } + + public static GreptimeSettings anonymous(GreptimeEndpoints endpoints, String database) { + return new GreptimeSettings(endpoints, database, Optional.empty()); + } + + public static GreptimeSettings authenticated(GreptimeEndpoints endpoints, String database, String username) { + if (username == null || username.isBlank()) { + throw new IllegalArgumentException("username must not be blank"); + } + return new GreptimeSettings(endpoints, database, Optional.of(username)); + } + + @Override + public String toString() { + return "GreptimeSettings[configured=true, authenticated=" + username.isPresent() + "]"; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedActiveConfigurationInspector.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedActiveConfigurationInspector.java new file mode 100644 index 0000000000..62c1c7523d --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedActiveConfigurationInspector.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.nio.file.Path; +import java.util.Map; +import java.util.Objects; + +/** Reads, verifies, and materializes the active pair exactly once for startup consumption. */ +public final class ManagedActiveConfigurationInspector { + + public static final String MANAGED_APPLICATION_SOURCE = "hertzbeatManagedApplication"; + public static final String MANAGED_SECRET_SOURCE = "hertzbeatManagedSecrets"; + + private final ManagedApplicationConfigStore applicationStore; + private final ManagedSecretStore secretStore; + + public ManagedActiveConfigurationInspector(Path installationRoot) { + this(new FileManagedApplicationConfigStore(installationRoot), + new FileManagedSecretStore(installationRoot)); + } + + ManagedActiveConfigurationInspector( + ManagedApplicationConfigStore applicationStore, ManagedSecretStore secretStore) { + this.applicationStore = Objects.requireNonNull(applicationStore, "applicationStore"); + this.secretStore = Objects.requireNonNull(secretStore, "secretStore"); + } + + public Inspection inspect() { + CandidateRead application = applicationStore.readActive(); + CandidateRead secrets = secretStore.readActive(); + if (application.state() == CandidateState.MISSING && secrets.state() == CandidateState.MISSING) { + return Inspection.absent(); + } + if (application.state() != CandidateState.VALID + || secrets.state() != CandidateState.VALID + || !application.generation().equals(secrets.generation())) { + return Inspection.recoveryRequired(); + } + try { + ManagedConfigurationBundle bundle = new ManagedConfigurationBundle( + application.value().orElseThrow(), secrets.value().orElseThrow()); + return Inspection.loadable( + ApplicationConfigDocumentCodec.springProperties(bundle.application()), + SecretConfigDocumentCodec.springProperties(bundle.secrets())); + } catch (IllegalArgumentException failure) { + return Inspection.recoveryRequired(); + } + } + + /** Immutable verified startup material; its string form never renders property values. */ + public record Inspection( + State state, + Map applicationProperties, + Map secretProperties) { + + public Inspection { + Objects.requireNonNull(state, "state"); + applicationProperties = Map.copyOf(applicationProperties); + secretProperties = Map.copyOf(secretProperties); + if (state != State.LOADABLE + && (!applicationProperties.isEmpty() || !secretProperties.isEmpty())) { + throw new IllegalArgumentException("Only loadable inspection may contain properties"); + } + } + + private static Inspection absent() { + return empty(State.ABSENT); + } + + private static Inspection recoveryRequired() { + return empty(State.RECOVERY_REQUIRED); + } + + private static Inspection loadable( + Map applicationProperties, + Map secretProperties) { + return new Inspection(State.LOADABLE, applicationProperties, secretProperties); + } + + private static Inspection empty(State state) { + return new Inspection(state, Map.of(), Map.of()); + } + + @Override + public String toString() { + return "Inspection[state=" + state + "]"; + } + } + + /** Startup-safe classification without exposing configuration or failure details. */ + public enum State { + ABSENT, + LOADABLE, + RECOVERY_REQUIRED + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedApplicationConfig.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedApplicationConfig.java new file mode 100644 index 0000000000..12cc1ccda6 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedApplicationConfig.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.util.Objects; + +/** Supported non-secret application overlay owned by setup. */ +public record ManagedApplicationConfig( + MetadataDatabaseSettings metadataDatabase, + GreptimeSettings telemetryStore) { + + public ManagedApplicationConfig { + Objects.requireNonNull(metadataDatabase, "metadataDatabase"); + Objects.requireNonNull(telemetryStore, "telemetryStore"); + } + + @Override + public String toString() { + return "ManagedApplicationConfig[configured=true]"; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedApplicationConfigStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedApplicationConfigStore.java new file mode 100644 index 0000000000..7c9e832e80 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedApplicationConfigStore.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.io.IOException; + +/** Persistence boundary for the supported non-secret managed overlay. */ +interface ManagedApplicationConfigStore { + + CandidateRead readCandidate(); + + CandidateRead readActive(); + + CandidateRead readLastKnownGood(); + + void stageCandidate(ManagedApplicationConfig candidate, String generation) throws IOException; + + void promoteCandidate(ManagedApplicationConfig expected, String generation) throws IOException; + + void restoreActive(CandidateRead previous) throws IOException; + + void discardCandidate() throws IOException; +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigCapability.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigCapability.java new file mode 100644 index 0000000000..569aa88b80 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigCapability.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; + +/** Deployment-aware configuration application capability. */ +public record ManagedConfigCapability( + ApplyMode applyMode, + boolean writableManagedConfig, + DeploymentConstraint constraint) { + + public ManagedConfigCapability { + Objects.requireNonNull(applyMode, "applyMode"); + Objects.requireNonNull(constraint, "constraint"); + if (writableManagedConfig != (applyMode == ApplyMode.MANAGED_WRITE) + || writableManagedConfig != (constraint == DeploymentConstraint.NONE)) { + throw new IllegalArgumentException("Managed write capability is inconsistent"); + } + } + + static ManagedConfigCapability writable() { + return new ManagedConfigCapability(ApplyMode.MANAGED_WRITE, true, DeploymentConstraint.NONE); + } + + static ManagedConfigCapability constrained(DeploymentConstraint constraint) { + return new ManagedConfigCapability(ApplyMode.EXTERNAL_APPLY, false, constraint); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigDeploymentDetector.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigDeploymentDetector.java new file mode 100644 index 0000000000..b43d009e62 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigDeploymentDetector.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.function.Predicate; + +/** Detects whether setup may safely use managed files or must export them for an operator. */ +public final class ManagedConfigDeploymentDetector { + + private static final List MANAGED_ARTIFACTS = List.of( + "managed-application.yml", + "managed-application.yml.candidate", + "managed-application.yml.last-known-good", + "managed-secrets.properties", + "managed-secrets.properties.candidate", + "managed-secrets.properties.last-known-good", + ".managed-config.lock"); + + private final Path installationRoot; + private final Predicate writable; + private final Predicate readable; + + public ManagedConfigDeploymentDetector(Path installationRoot) { + this(installationRoot, Files::isWritable, Files::isReadable); + } + + ManagedConfigDeploymentDetector(Path installationRoot, Predicate writable) { + this(installationRoot, writable, Files::isReadable); + } + + ManagedConfigDeploymentDetector( + Path installationRoot, Predicate writable, Predicate readable) { + this.installationRoot = installationRoot.toAbsolutePath().normalize(); + this.writable = writable; + this.readable = readable; + } + + public ManagedConfigCapability detect() { + if (!Files.exists(installationRoot)) { + return ManagedConfigCapability.constrained(DeploymentConstraint.INSTALLATION_ROOT_MISSING); + } + if (!Files.isDirectory(installationRoot)) { + return ManagedConfigCapability.constrained(DeploymentConstraint.INSTALLATION_ROOT_NOT_DIRECTORY); + } + Path configDirectory = installationRoot.resolve("data/config"); + if (hasManagedSymlink(configDirectory)) { + return ManagedConfigCapability.constrained(DeploymentConstraint.UNSAFE_PATH); + } + Path existingParent = nearestExisting(configDirectory); + if (!Files.isDirectory(existingParent)) { + return ManagedConfigCapability.constrained(DeploymentConstraint.CONFIG_PATH_NOT_DIRECTORY); + } + if (!writable.test(existingParent)) { + return ManagedConfigCapability.constrained(DeploymentConstraint.READ_ONLY); + } + for (String fileName : MANAGED_ARTIFACTS) { + Path managedFile = configDirectory.resolve(fileName); + if (Files.exists(managedFile) && !Files.isRegularFile(managedFile)) { + return ManagedConfigCapability.constrained(DeploymentConstraint.UNSAFE_PATH); + } + if (Files.exists(managedFile) + && (!readable.test(managedFile) || !writable.test(managedFile))) { + return ManagedConfigCapability.constrained(DeploymentConstraint.READ_ONLY); + } + } + return ManagedConfigCapability.writable(); + } + + private Path nearestExisting(Path path) { + Path current = path; + while (!Files.exists(current) && !current.equals(installationRoot)) { + current = current.getParent(); + } + return current; + } + + private boolean hasManagedSymlink(Path configDirectory) { + if (Files.isSymbolicLink(installationRoot) + || Files.isSymbolicLink(installationRoot.resolve("data")) + || Files.isSymbolicLink(configDirectory)) { + return true; + } + for (String fileName : MANAGED_ARTIFACTS) { + if (Files.isSymbolicLink(configDirectory.resolve(fileName))) { + return true; + } + } + return false; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationBundle.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationBundle.java new file mode 100644 index 0000000000..7a5241cea5 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationBundle.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.util.Objects; + +/** The only valid unit for applying or loading the separate application and secret documents. */ +public record ManagedConfigurationBundle(ManagedApplicationConfig application, ManagedSecrets secrets) { + + public ManagedConfigurationBundle { + Objects.requireNonNull(application, "application"); + Objects.requireNonNull(secrets, "secrets"); + boolean telemetryUsername = application.telemetryStore().username().isPresent(); + boolean telemetryPassword = secrets.telemetryPassword().isPresent(); + if (telemetryUsername != telemetryPassword) { + throw new IllegalArgumentException("Telemetry username and password must be configured together"); + } + } + + @Override + public String toString() { + return "ManagedConfigurationBundle[configured=true, secrets=]"; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransaction.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransaction.java new file mode 100644 index 0000000000..4ec61540ee --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransaction.java @@ -0,0 +1,304 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Objects; +import java.util.UUID; + +/** Coordinates the application and secret snapshots as one locked, recoverable operation. */ +public final class ManagedConfigurationTransaction { + + private static final String LOCK_FILE = ".managed-config.lock"; + + private final ManagedApplicationConfigStore applicationStore; + private final ManagedSecretStore secretStore; + private final Path lockFile; + + /** Creates the production file transaction rooted at the HertzBeat installation. */ + public ManagedConfigurationTransaction(Path installationRoot) { + this(new FileManagedApplicationConfigStore(installationRoot), + new FileManagedSecretStore(installationRoot), installationRoot); + } + + ManagedConfigurationTransaction( + ManagedApplicationConfigStore applicationStore, + ManagedSecretStore secretStore, + Path installationRoot) { + this.applicationStore = Objects.requireNonNull(applicationStore, "applicationStore"); + this.secretStore = Objects.requireNonNull(secretStore, "secretStore"); + Path root = Objects.requireNonNull(installationRoot, "installationRoot") + .toAbsolutePath().normalize(); + this.lockFile = root.resolve("data/config").resolve(LOCK_FILE); + } + + /** Stages and publishes one validated configuration generation under the process lock. */ + public Outcome apply(ManagedConfigurationBundle bundle) throws IOException { + Objects.requireNonNull(bundle, "bundle"); + return withLock(() -> applyLocked(bundle)); + } + + /** Converges interrupted publication only when an explicit complete generation pair exists. */ + public Outcome recover() throws IOException { + return withLock(this::recoverLocked); + } + + private Outcome applyLocked(ManagedConfigurationBundle bundle) throws IOException { + CandidateRead previousApplication = applicationStore.readActive(); + CandidateRead previousSecrets = secretStore.readActive(); + if (!formsPair(previousApplication, previousSecrets)) { + return Outcome.RECOVERY_REQUIRED; + } + String generation = UUID.randomUUID().toString(); + try { + applicationStore.stageCandidate(bundle.application(), generation); + secretStore.stageCandidate(bundle.secrets(), generation); + } catch (IOException failure) { + discardCandidate(applicationStore, failure); + discardCandidate(secretStore, failure); + throw failure; + } + CandidateRead applicationCandidate = applicationStore.readCandidate(); + CandidateRead secretCandidate = secretStore.readCandidate(); + if (!sameGeneration(applicationCandidate, secretCandidate, generation)) { + return discardCandidates() ? Outcome.NOT_APPLIED : Outcome.RECOVERY_REQUIRED; + } + try { + applicationStore.promoteCandidate(bundle.application(), generation); + secretStore.promoteCandidate(bundle.secrets(), generation); + if (!matchesExpectedActive(bundle, generation)) { + return rollbackAfterPromotionFailure(previousApplication, previousSecrets); + } + return Outcome.APPLIED; + } catch (IOException ignored) { + return rollbackAfterPromotionFailure(previousApplication, previousSecrets); + } + } + + private Outcome rollbackAfterPromotionFailure( + CandidateRead previousApplication, + CandidateRead previousSecrets) { + boolean applicationRestored = restoreApplication(previousApplication); + boolean secretsRestored = restoreSecrets(previousSecrets); + if (!applicationRestored || !secretsRestored) { + return Outcome.RECOVERY_REQUIRED; + } + return discardCandidates() ? Outcome.ROLLED_BACK : Outcome.RECOVERY_REQUIRED; + } + + private Outcome recoverLocked() { + Snapshots applications = new Snapshots<>( + applicationStore.readActive(), applicationStore.readCandidate(), + applicationStore.readLastKnownGood()); + Snapshots secrets = new Snapshots<>( + secretStore.readActive(), secretStore.readCandidate(), secretStore.readLastKnownGood()); + + if (formsPair(applications.active(), secrets.active())) { + boolean interrupted = applications.candidate().state() != CandidateState.MISSING + || secrets.candidate().state() != CandidateState.MISSING; + return discardCandidates() + ? (interrupted ? Outcome.ROLLED_BACK : Outcome.APPLIED) + : Outcome.RECOVERY_REQUIRED; + } + + return recoverExplicitPair(applications, secrets); + } + + private Outcome recoverExplicitPair( + Snapshots applications, Snapshots secrets) { + // Crash-state invariant (no generation ordering is inferred): active+candidate and + // candidate+active are the only split-promotion roll-forwards; LKG participates only + // in a complete same-generation rollback pair. Any other shape remains recovery-required. + if (validPair(applications.active(), secrets.candidate())) { + return finishRecovery(promoteSecrets(secrets.candidate()), Outcome.APPLIED); + } + if (validPair(applications.candidate(), secrets.active())) { + return finishRecovery(promoteApplication(applications.candidate()), Outcome.APPLIED); + } + if (validPair(applications.lastKnownGood(), secrets.active())) { + return finishRecovery( + restoreApplication(applications.lastKnownGood()), Outcome.ROLLED_BACK); + } + if (validPair(applications.active(), secrets.lastKnownGood())) { + return finishRecovery( + restoreSecrets(secrets.lastKnownGood()), Outcome.ROLLED_BACK); + } + if (validPair(applications.lastKnownGood(), secrets.lastKnownGood())) { + return finishRecovery(restoreBoth( + applications.lastKnownGood(), secrets.lastKnownGood()), Outcome.ROLLED_BACK); + } + return Outcome.RECOVERY_REQUIRED; + } + + private Outcome finishRecovery(boolean recovered, Outcome outcome) { + boolean candidatesDiscarded = discardCandidates(); + return recovered && candidatesDiscarded ? outcome : Outcome.RECOVERY_REQUIRED; + } + + private boolean restoreBoth( + CandidateRead application, + CandidateRead secrets) { + boolean applicationRestored = restoreApplication(application); + boolean secretsRestored = restoreSecrets(secrets); + return applicationRestored && secretsRestored; + } + + private boolean promoteApplication(CandidateRead candidate) { + try { + applicationStore.promoteCandidate( + candidate.value().orElseThrow(), candidate.generation().orElseThrow()); + return true; + } catch (IOException failure) { + return false; + } + } + + private boolean promoteSecrets(CandidateRead candidate) { + try { + secretStore.promoteCandidate( + candidate.value().orElseThrow(), candidate.generation().orElseThrow()); + return true; + } catch (IOException failure) { + return false; + } + } + + private boolean restoreApplication(CandidateRead previous) { + try { + applicationStore.restoreActive(previous); + return true; + } catch (IOException failure) { + return false; + } + } + + private boolean restoreSecrets(CandidateRead previous) { + try { + secretStore.restoreActive(previous); + return true; + } catch (IOException failure) { + return false; + } + } + + private boolean discardCandidates() { + boolean discarded = true; + try { + applicationStore.discardCandidate(); + } catch (IOException failure) { + discarded = false; + } + try { + secretStore.discardCandidate(); + } catch (IOException failure) { + discarded = false; + } + return discarded; + } + + private Outcome withLock(LockedOperation operation) throws IOException { + Path directory = lockFile.getParent(); + if (Files.isSymbolicLink(lockFile) || Files.isSymbolicLink(directory) + || Files.isSymbolicLink(directory.getParent()) + || Files.isSymbolicLink(directory.getParent().getParent())) { + throw new IOException("Managed configuration lock is unavailable"); + } + Files.createDirectories(directory); + try (FileChannel channel = FileChannel.open(lockFile, + StandardOpenOption.CREATE, StandardOpenOption.WRITE); + FileLock lock = tryLock(channel)) { + if (lock == null) { + throw new IOException("Managed configuration operation is already in progress"); + } + return operation.run(); + } + } + + private static FileLock tryLock(FileChannel channel) throws IOException { + try { + return channel.tryLock(); + } catch (OverlappingFileLockException failure) { + return null; + } + } + + private static void discardCandidate(ManagedApplicationConfigStore store, IOException failure) { + try { + store.discardCandidate(); + } catch (IOException cleanupFailure) { + failure.addSuppressed(cleanupFailure); + } + } + + private static void discardCandidate(ManagedSecretStore store, IOException failure) { + try { + store.discardCandidate(); + } catch (IOException cleanupFailure) { + failure.addSuppressed(cleanupFailure); + } + } + + private static boolean sameGeneration( + CandidateRead left, CandidateRead right, String generation) { + return validPair(left, right) && left.generation().filter(generation::equals).isPresent(); + } + + private boolean matchesExpectedActive(ManagedConfigurationBundle bundle, String generation) { + CandidateRead application = applicationStore.readActive(); + CandidateRead secrets = secretStore.readActive(); + return sameGeneration(application, secrets, generation) + && application.value().filter(bundle.application()::equals).isPresent() + && secrets.value().filter(bundle.secrets()::equals).isPresent(); + } + + private static boolean formsPair(CandidateRead left, CandidateRead right) { + if (left.state() == CandidateState.MISSING && right.state() == CandidateState.MISSING) { + return true; + } + return validPair(left, right); + } + + private static boolean validPair(CandidateRead left, CandidateRead right) { + return left.state() == CandidateState.VALID + && right.state() == CandidateState.VALID + && left.generation().equals(right.generation()); + } + + private record Snapshots(CandidateRead active, CandidateRead candidate, + CandidateRead lastKnownGood) { + } + + @FunctionalInterface + private interface LockedOperation { + Outcome run() throws IOException; + } + + /** Stable outcome without exception details or secret content. */ + public enum Outcome { + APPLIED, + NOT_APPLIED, + ROLLED_BACK, + RECOVERY_REQUIRED + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedDocumentCodec.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedDocumentCodec.java new file mode 100644 index 0000000000..72d8a66760 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedDocumentCodec.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; +import java.util.Locale; +import java.util.Objects; +import java.util.regex.Pattern; + +interface ManagedDocumentCodec { + + byte[] encode(T value, String generation); + + Decoded decode(byte[] content) throws DocumentException; + + record Decoded(T value, String generation) { + + public Decoded { + Objects.requireNonNull(value, "value"); + Integrity.requireValidGeneration(generation); + } + } + + final class DocumentException extends Exception { + + private final CandidateState state; + + private DocumentException(CandidateState state) { + super("Managed configuration is " + state.name().toLowerCase(Locale.ROOT)); + this.state = state; + } + + static DocumentException invalid() { + return new DocumentException(CandidateState.INVALID); + } + + static DocumentException corrupt() { + return new DocumentException(CandidateState.CORRUPT); + } + + CandidateState state() { + return state; + } + } + + final class Integrity { + + static final String FORMAT_HEADER = "# hertzbeat-managed-format: 1\n"; + static final String GENERATION_PREFIX = "# hertzbeat-managed-generation: "; + static final String CHECKSUM_PREFIX = "# hertzbeat-managed-sha256: "; + private static final Pattern GENERATION = Pattern.compile("[A-Za-z0-9-]{1,64}"); + + private Integrity() { + } + + static byte[] envelope(String body, String generation) { + requireValidGeneration(generation); + String protectedContent = generation + "\n" + body; + String document = FORMAT_HEADER + GENERATION_PREFIX + generation + "\n" + + CHECKSUM_PREFIX + checksum(protectedContent) + "\n" + body; + return document.getBytes(StandardCharsets.UTF_8); + } + + static VerifiedBody extract(byte[] content) throws DocumentException { + String document = new String(content, StandardCharsets.UTF_8); + if (!document.startsWith(FORMAT_HEADER + GENERATION_PREFIX)) { + throw DocumentException.corrupt(); + } + int generationStart = FORMAT_HEADER.length() + GENERATION_PREFIX.length(); + int generationEnd = document.indexOf('\n', generationStart); + if (generationEnd < 0) { + throw DocumentException.corrupt(); + } + String generation = document.substring(generationStart, generationEnd); + if (!GENERATION.matcher(generation).matches()) { + throw DocumentException.corrupt(); + } + int checksumPrefixStart = generationEnd + 1; + if (!document.startsWith(CHECKSUM_PREFIX, checksumPrefixStart)) { + throw DocumentException.corrupt(); + } + int checksumStart = checksumPrefixStart + CHECKSUM_PREFIX.length(); + int checksumEnd = document.indexOf('\n', checksumStart); + if (checksumEnd < 0) { + throw DocumentException.corrupt(); + } + return new VerifiedBody(generation, document.substring(checksumStart, checksumEnd), + document.substring(checksumEnd + 1)); + } + + static void verify(VerifiedBody body) throws DocumentException { + String protectedContent = body.generation() + "\n" + body.content(); + if (!MessageDigest.isEqual( + body.expectedChecksum().getBytes(StandardCharsets.US_ASCII), + checksum(protectedContent).getBytes(StandardCharsets.US_ASCII))) { + throw DocumentException.corrupt(); + } + } + + static void requireValidGeneration(String generation) { + Objects.requireNonNull(generation, "generation"); + if (!GENERATION.matcher(generation).matches()) { + throw new IllegalArgumentException("Invalid managed configuration generation"); + } + } + + static String literalForSpring(String value) { + return value.replace("${", "\\${"); + } + + private static String checksum(String content) { + try { + return HexFormat.of().formatHex( + MessageDigest.getInstance("SHA-256").digest(content.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException impossible) { + throw new IllegalStateException("SHA-256 is unavailable", impossible); + } + } + + record VerifiedBody(String generation, String expectedChecksum, String content) { + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedFileIo.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedFileIo.java new file mode 100644 index 0000000000..86d553c47a --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedFileIo.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.io.IOException; +import java.nio.file.Path; + +/** Narrow I/O seams used by the durable file adapter and its failure-path tests. */ +final class ManagedFileIo { + + private ManagedFileIo() { + } + + interface Publisher { + + void publish(Path target, byte[] content, boolean ownerOnly) throws IOException; + + void remove(Path target) throws IOException; + } + + @FunctionalInterface + interface Reader { + + byte[] read(Path path) throws IOException; + } + + interface Operations { + + void atomicReplace(Path source, Path target) throws IOException; + + void forceDirectory(Path directory) throws IOException; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedSecretStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedSecretStore.java new file mode 100644 index 0000000000..17b9f8f249 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedSecretStore.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.io.IOException; + +/** Persistence boundary for setup-owned secrets. */ +interface ManagedSecretStore { + + CandidateRead readCandidate(); + + CandidateRead readActive(); + + CandidateRead readLastKnownGood(); + + void stageCandidate(ManagedSecrets candidate, String generation) throws IOException; + + void promoteCandidate(ManagedSecrets expected, String generation) throws IOException; + + void restoreActive(CandidateRead previous) throws IOException; + + void discardCandidate() throws IOException; +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedSecrets.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedSecrets.java new file mode 100644 index 0000000000..a492f32d2d --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedSecrets.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.util.Objects; +import java.util.Optional; + +/** Setup-owned secrets stored outside the managed application overlay. */ +public record ManagedSecrets(SecretValue metadataDatabasePassword, Optional telemetryPassword) { + + public ManagedSecrets { + Objects.requireNonNull(metadataDatabasePassword, "metadataDatabasePassword"); + Objects.requireNonNull(telemetryPassword, "telemetryPassword"); + } + + public static ManagedSecrets withoutTelemetryPassword(SecretValue metadataDatabasePassword) { + return new ManagedSecrets(metadataDatabasePassword, Optional.empty()); + } + + public static ManagedSecrets withTelemetryPassword( + SecretValue metadataDatabasePassword, SecretValue telemetryPassword) { + return new ManagedSecrets(metadataDatabasePassword, Optional.of(telemetryPassword)); + } + + @Override + public String toString() { + return "ManagedSecrets[metadataDatabasePassword=, telemetryPassword=]"; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MetadataDatabaseSettings.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MetadataDatabaseSettings.java new file mode 100644 index 0000000000..c724af9f4c --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MetadataDatabaseSettings.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Supported non-secret metadata database settings. */ +public record MetadataDatabaseSettings(MetadataDatabaseKind kind, String jdbcUrl, String username) { + + public MetadataDatabaseSettings { + Objects.requireNonNull(kind, "kind"); + requireText(jdbcUrl, "jdbcUrl"); + requireText(username, "username"); + } + + @Override + public String toString() { + return "MetadataDatabaseSettings[kind=" + kind + ", configured=true]"; + } + + private static void requireText(String value, String name) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException(name + " must not be blank"); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/NioManagedFilePublisher.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/NioManagedFilePublisher.java new file mode 100644 index 0000000000..75edf8fe6a --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/NioManagedFilePublisher.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.PosixFilePermission; +import java.util.Set; + +/** Durable temp-write, file-fsync, replace, and directory-fsync publication. */ +final class NioManagedFilePublisher implements ManagedFileIo.Publisher { + + private static final Set OWNER_ONLY = Set.of( + PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE); + + private final ManagedFileIo.Operations operations; + + NioManagedFilePublisher() { + this(new NioOperations()); + } + + NioManagedFilePublisher(ManagedFileIo.Operations operations) { + this.operations = operations; + } + + @Override + public void publish(Path target, byte[] content, boolean ownerOnly) throws IOException { + Path directory = target.toAbsolutePath().getParent(); + Files.createDirectories(directory); + Path temporary = Files.createTempFile(directory, ".managed-config-", ".tmp"); + try { + if (ownerOnly) { + setOwnerOnlyWhenSupported(temporary); + } + try (FileChannel channel = FileChannel.open( + temporary, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) { + ByteBuffer buffer = ByteBuffer.wrap(content); + while (buffer.hasRemaining()) { + channel.write(buffer); + } + channel.force(true); + } + replaceAndForce(temporary, target); + } finally { + Files.deleteIfExists(temporary); + } + } + + @Override + public void remove(Path target) throws IOException { + if (Files.deleteIfExists(target)) { + operations.forceDirectory(target.toAbsolutePath().getParent()); + } + } + + private void replaceAndForce(Path source, Path target) throws IOException { + operations.atomicReplace(source, target); + operations.forceDirectory(target.toAbsolutePath().getParent()); + } + + private static void setOwnerOnlyWhenSupported(Path path) throws IOException { + if (Files.getFileStore(path).supportsFileAttributeView("posix")) { + Files.setPosixFilePermissions(path, OWNER_ONLY); + } + } + + private static final class NioOperations implements ManagedFileIo.Operations { + + @Override + public void atomicReplace(Path source, Path target) throws IOException { + Files.move(source, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } + + @Override + public void forceDirectory(Path directory) throws IOException { + if (!Files.getFileStore(directory).supportsFileAttributeView("posix")) { + return; + } + try (FileChannel channel = FileChannel.open(directory, StandardOpenOption.READ)) { + channel.force(true); + } + } + } + +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/RestartRequirement.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/RestartRequirement.java new file mode 100644 index 0000000000..b37f1dafac --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/RestartRequirement.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +/** Activation behavior attached to one effective configuration value. */ +public enum RestartRequirement { + LIVE_RELOAD, + RESTART_REQUIRED +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SecretConfigDocumentCodec.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SecretConfigDocumentCodec.java new file mode 100644 index 0000000000..efe0328b77 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SecretConfigDocumentCodec.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.io.IOException; +import java.io.StringReader; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +final class SecretConfigDocumentCodec implements ManagedDocumentCodec { + + private static final String METADATA_PASSWORD = "spring.datasource.password"; + private static final String TELEMETRY_PASSWORD = "warehouse.store.greptime.password"; + + @Override + public byte[] encode(ManagedSecrets value, String generation) { + StringBuilder body = new StringBuilder(); + append(body, METADATA_PASSWORD, value.metadataDatabasePassword()); + value.telemetryPassword().ifPresent(secret -> append(body, TELEMETRY_PASSWORD, secret)); + return Integrity.envelope(body.toString(), generation); + } + + static Map springProperties(ManagedSecrets value) { + Map properties = new LinkedHashMap<>(); + properties.put(METADATA_PASSWORD, springLiteral(value.metadataDatabasePassword())); + value.telemetryPassword().ifPresent( + secret -> properties.put(TELEMETRY_PASSWORD, springLiteral(secret))); + return Map.copyOf(properties); + } + + @Override + public Decoded decode(byte[] content) + throws DocumentException { + Integrity.VerifiedBody body = Integrity.extract(content); + Integrity.verify(body); + Properties properties = new Properties(); + try { + properties.load(new StringReader(body.content())); + } catch (IOException | IllegalArgumentException exception) { + throw DocumentException.corrupt(); + } + if (!properties.stringPropertyNames().contains(METADATA_PASSWORD) + || !Set.of(METADATA_PASSWORD, TELEMETRY_PASSWORD).containsAll(properties.stringPropertyNames())) { + throw DocumentException.corrupt(); + } + ManagedSecrets decoded; + try { + SecretValue metadata = SecretValue.of( + removeSpringPlaceholderEscapes(properties.getProperty(METADATA_PASSWORD))); + decoded = properties.containsKey(TELEMETRY_PASSWORD) + ? ManagedSecrets.withTelemetryPassword( + metadata, SecretValue.of(removeSpringPlaceholderEscapes( + properties.getProperty(TELEMETRY_PASSWORD)))) + : ManagedSecrets.withoutTelemetryPassword(metadata); + } catch (IllegalArgumentException exception) { + throw DocumentException.invalid(); + } + return new Decoded<>(decoded, body.generation()); + } + + private static void append(StringBuilder body, String key, SecretValue secret) { + char[] copy = secret.copy(); + try { + body.append(key).append('=').append(escape(copy)).append('\n'); + } finally { + java.util.Arrays.fill(copy, '\0'); + } + } + + private static String escape(char[] value) { + StringBuilder escaped = new StringBuilder(value.length); + for (int index = 0; index < value.length; index++) { + char character = value[index]; + switch (character) { + case '\\' -> escaped.append("\\\\"); + case '\n' -> escaped.append("\\n"); + case '\r' -> escaped.append("\\r"); + case '\t' -> escaped.append("\\t"); + case '\f' -> escaped.append("\\f"); + case '=', ':' -> escaped.append('\\').append(character); + case ' ' -> escaped.append(index == 0 ? "\\ " : " "); + case '$' -> { + if (index + 1 < value.length && value[index + 1] == '{') { + // Properties decoding keeps one slash, which makes Spring treat ${...} literally. + escaped.append("\\\\"); + } + escaped.append(character); + } + default -> escaped.append(character); + } + } + return escaped.toString(); + } + + private static String removeSpringPlaceholderEscapes(String value) { + StringBuilder decoded = new StringBuilder(value.length()); + for (int index = 0; index < value.length(); index++) { + char character = value.charAt(index); + if (character == '\\' && index + 2 < value.length() + && value.charAt(index + 1) == '$' && value.charAt(index + 2) == '{') { + continue; + } + decoded.append(character); + } + return decoded.toString(); + } + + private static String springLiteral(SecretValue secret) { + char[] copy = secret.copy(); + try { + return Integrity.literalForSpring(new String(copy)); + } finally { + java.util.Arrays.fill(copy, '\0'); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SecretValue.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SecretValue.java new file mode 100644 index 0000000000..f471db13e9 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SecretValue.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.util.Arrays; + +/** Mutable-copy-resistant secret value with redacted diagnostics. */ +public final class SecretValue { + + private final char[] content; + + private SecretValue(char[] content) { + this.content = content.clone(); + } + + public static SecretValue of(String content) { + if (content == null || content.isBlank()) { + throw new IllegalArgumentException("Secret value must not be blank"); + } + return new SecretValue(content.toCharArray()); + } + + public char[] copy() { + return content.clone(); + } + + @Override + public boolean equals(Object other) { + return other instanceof SecretValue secret && Arrays.equals(content, secret.content); + } + + @Override + public int hashCode() { + return Arrays.hashCode(content); + } + + @Override + public String toString() { + return "SecretValue[]"; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SensitiveExportContent.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SensitiveExportContent.java new file mode 100644 index 0000000000..49642c202d --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SensitiveExportContent.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +/** Defensive secret-bearing export bytes that never render their content. */ +public final class SensitiveExportContent { + + private final byte[] content; + + private SensitiveExportContent(byte[] content) { + this.content = content.clone(); + } + + public static SensitiveExportContent of(byte[] content) { + if (content == null || content.length == 0) { + throw new IllegalArgumentException("Export content must not be empty"); + } + return new SensitiveExportContent(content); + } + + public byte[] copy() { + return content.clone(); + } + + @Override + public String toString() { + return "SensitiveExportContent[]"; + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java index 45f93a2241..1f364b6aeb 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java @@ -19,6 +19,7 @@ package org.apache.hertzbeat.manager.setup.api; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import com.fasterxml.jackson.databind.ObjectMapper; @@ -160,6 +161,22 @@ class SetupApiContractTest { ValidationSection.MAIL, metadata, null, null, null)); } + @Test + void greptimeCredentialsAreOptionalButMustBeSuppliedTogether() { + TelemetryStoreConfiguration anonymous = new TelemetryStoreConfiguration( + TelemetryStoreKind.GREPTIME, "greptime:4001", "http://greptime:4000", "public", null, null); + + assertNull(anonymous.username()); + TelemetryStoreConfiguration blank = new TelemetryStoreConfiguration( + TelemetryStoreKind.GREPTIME, "greptime:4001", "http://greptime:4000", "public", " ", "\t"); + assertNull(blank.username()); + assertNull(blank.password()); + assertThrows(IllegalArgumentException.class, () -> new TelemetryStoreConfiguration( + TelemetryStoreKind.GREPTIME, "greptime:4001", "http://greptime:4000", "public", "user", null)); + assertThrows(IllegalArgumentException.class, () -> new TelemetryStoreConfiguration( + TelemetryStoreKind.GREPTIME, "greptime:4001", "http://greptime:4000", "public", null, SECRET)); + } + @Test void freezesStableSafeErrorCodes() throws Exception { assertWireValues(SetupErrorCode.values(), "setup_complete", "setup_locked", "setup_code_invalid", diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/EffectiveConfigurationResolverTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/EffectiveConfigurationResolverTest.java new file mode 100644 index 0000000000..4498b8089b --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/EffectiveConfigurationResolverTest.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.Map; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigSource; +import org.junit.jupiter.api.Test; +import org.springframework.core.env.MapPropertySource; +import org.springframework.core.env.StandardEnvironment; + +class EffectiveConfigurationResolverTest { + + private static final String KEY = "sample.key"; + private final EffectiveConfigurationResolver resolver = new EffectiveConfigurationResolver(); + + @Test + void resolvesValueSourceAndRestartMetadataFromSpringEnvironment() { + StandardEnvironment environment = new StandardEnvironment(); + environment.getPropertySources().addFirst(new MapPropertySource( + StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME, Map.of(KEY, "selected"))); + + EffectiveConfigurationValue result = + resolver.resolve(environment, KEY, RestartRequirement.RESTART_REQUIRED); + + assertEquals("selected", result.value()); + assertEquals(ConfigSource.SYSTEM_PROPERTY, result.source()); + assertEquals(RestartRequirement.RESTART_REQUIRED, result.restartRequirement()); + } + + @Test + void rejectsUnknownPropertySourcesInsteadOfMisclassifyingThem() { + StandardEnvironment environment = new StandardEnvironment(); + environment.getPropertySources().addFirst(new MapPropertySource("unsupported", Map.of(KEY, "value"))); + + assertThrows(IllegalStateException.class, + () -> resolver.resolve(environment, KEY, RestartRequirement.LIVE_RELOAD)); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ExternalConfigExportArtifactTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ExternalConfigExportArtifactTest.java new file mode 100644 index 0000000000..cdfe35f26e --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ExternalConfigExportArtifactTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.charset.StandardCharsets; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; +import org.junit.jupiter.api.Test; + +class ExternalConfigExportArtifactTest { + + private static final String SECRET = "export-secret-value"; + + @Test + void carriesSensitiveContentWithoutClaimingItWasApplied() { + byte[] source = SECRET.getBytes(StandardCharsets.UTF_8); + ExternalConfigExportArtifact artifact = new ExternalConfigExportArtifact( + "hertzbeat-managed.env", "text/plain", SensitiveExportContent.of(source)); + source[0] = 'x'; + + assertArrayEquals(SECRET.getBytes(StandardCharsets.UTF_8), artifact.content().copy()); + assertEquals(SetupOperationState.AWAITING_EXTERNAL_APPLY, artifact.state()); + assertTrue(artifact.noStore()); + assertFalse(artifact.toString().contains(SECRET)); + assertFalse(artifact.content().toString().contains(SECRET)); + } + + @Test + void rejectsUnsafeAttachmentNames() { + SensitiveExportContent content = SensitiveExportContent.of("safe".getBytes(StandardCharsets.UTF_8)); + + assertThrows(IllegalArgumentException.class, + () -> new ExternalConfigExportArtifact("../managed.env", "text/plain", content)); + assertThrows(IllegalArgumentException.class, + () -> new ExternalConfigExportArtifact("managed/env", "text/plain", content)); + assertThrows(IllegalArgumentException.class, + () -> new ExternalConfigExportArtifact("managed\\env", "text/plain", content)); + assertThrows(IllegalArgumentException.class, + () -> new ExternalConfigExportArtifact("managed\r\nenv", "text/plain", content)); + assertThrows(IllegalArgumentException.class, + () -> new ExternalConfigExportArtifact("managed.env", "text/plain\r\nx-test: value", content)); + assertThrows(IllegalArgumentException.class, + () -> new ExternalConfigExportArtifact("managed.env", "text\\plain", content)); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/FileManagedConfigurationStoreTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/FileManagedConfigurationStoreTest.java new file mode 100644 index 0000000000..63283bce62 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/FileManagedConfigurationStoreTest.java @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermission; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class FileManagedConfigurationStoreTest { + + private static final String DATABASE_PASSWORD = "database-secret-value"; + private static final String TELEMETRY_PASSWORD = "telemetry-secret-value"; + private static final String GENERATION = "test-generation"; + + @TempDir + private Path installationRoot; + + @Test + void stagesIsolatedTypedCandidatesAndKeepsSecretsSeparate() throws Exception { + FileManagedApplicationConfigStore applicationStore = new FileManagedApplicationConfigStore(installationRoot); + FileManagedSecretStore secretStore = new FileManagedSecretStore(installationRoot); + + applicationStore.stageCandidate(configuration("candidate"), GENERATION); + secretStore.stageCandidate(secrets(), GENERATION); + + Path configDirectory = installationRoot.resolve("data/config"); + try (Stream files = Files.list(configDirectory)) { + assertEquals(List.of("managed-application.yml.candidate", "managed-secrets.properties.candidate"), + files.map(path -> path.getFileName().toString()).sorted().toList()); + } + String applicationDocument = Files.readString( + configDirectory.resolve("managed-application.yml.candidate"), StandardCharsets.UTF_8); + assertFalse(applicationDocument.contains(DATABASE_PASSWORD)); + assertFalse(applicationDocument.contains(TELEMETRY_PASSWORD)); + assertEquals(configuration("candidate"), applicationStore.readCandidate().value().orElseThrow()); + assertEquals(secrets(), secretStore.readCandidate().value().orElseThrow()); + assertEquals(CandidateState.MISSING, applicationStore.readActive().state()); + assertEquals(CandidateState.MISSING, secretStore.readActive().state()); + } + + @Test + void promotesAndRetainsTheLastKnownGoodSnapshotForTransactionRecovery() throws Exception { + FileManagedApplicationConfigStore store = new FileManagedApplicationConfigStore(installationRoot); + ManagedApplicationConfig knownGood = configuration("known-good"); + ManagedApplicationConfig second = configuration("second"); + + store.stageCandidate(knownGood, "known-good-generation"); + store.promoteCandidate(knownGood, "known-good-generation"); + assertEquals(knownGood, store.readActive().value().orElseThrow()); + assertEquals(CandidateState.MISSING, store.readLastKnownGood().state()); + store.stageCandidate(second, "second-generation"); + store.promoteCandidate(second, "second-generation"); + + assertEquals(second, store.readActive().value().orElseThrow()); + assertEquals(knownGood, store.readLastKnownGood().value().orElseThrow()); + } + + @Test + void publishesTheVerifiedCandidateBytesWhenThePathIsReplacedAfterReading() throws Exception { + ManagedApplicationConfig expected = configuration("expected"); + ManagedApplicationConfig replacement = configuration("replacement"); + FileManagedApplicationConfigStore initial = new FileManagedApplicationConfigStore(installationRoot); + initial.stageCandidate(expected, GENERATION); + Path candidate = installationRoot.resolve("data/config/managed-application.yml.candidate"); + AtomicInteger candidateReads = new AtomicInteger(); + ManagedFileIo.Reader replacingReader = path -> { + byte[] verifiedBytes = Files.readAllBytes(path); + if (path.equals(candidate) && candidateReads.incrementAndGet() == 1) { + Files.write(path, new ApplicationConfigDocumentCodec().encode( + replacement, "replacement-generation")); + } + return verifiedBytes; + }; + FileManagedApplicationConfigStore store = new FileManagedApplicationConfigStore( + installationRoot, new NioManagedFilePublisher(), replacingReader); + + store.promoteCandidate(expected, GENERATION); + + assertEquals(1, candidateReads.get()); + assertEquals(expected, store.readActive().value().orElseThrow()); + assertEquals(CandidateState.MISSING, store.readCandidate().state()); + } + + @Test + void rejectsCandidateWithUnexpectedValueOrGenerationBeforeChangingActive() throws Exception { + ManagedApplicationConfig candidate = configuration("candidate"); + FileManagedApplicationConfigStore store = new FileManagedApplicationConfigStore(installationRoot); + store.stageCandidate(candidate, GENERATION); + + assertThrows(IOException.class, + () -> store.promoteCandidate(configuration("unexpected"), GENERATION)); + assertEquals(CandidateState.MISSING, store.readActive().state()); + assertEquals(candidate, store.readCandidate().value().orElseThrow()); + + assertThrows(IOException.class, + () -> store.promoteCandidate(candidate, "unexpected-generation")); + assertEquals(CandidateState.MISSING, store.readActive().state()); + assertEquals(candidate, store.readCandidate().value().orElseThrow()); + } + + @Test + void classifiesMissingInvalidCorruptAndUnreadableCandidates() throws Exception { + FileManagedApplicationConfigStore store = new FileManagedApplicationConfigStore(installationRoot); + Path managedFile = installationRoot.resolve("data/config/managed-application.yml.candidate"); + assertEquals(CandidateState.MISSING, store.readCandidate().state()); + + store.stageCandidate(configuration("valid"), GENERATION); + ManagedDocumentCodec.Integrity.VerifiedBody validBody = ManagedDocumentCodec.Integrity.extract( + Files.readAllBytes(managedFile)); + Files.write(managedFile, ManagedDocumentCodec.Integrity.envelope( + validBody.content().replace("database: 'public'", "database: ' '"), validBody.generation())); + assertEquals(CandidateState.INVALID, store.readCandidate().state()); + + Files.writeString(managedFile, "formatVersion: [broken", StandardCharsets.UTF_8); + assertEquals(CandidateState.CORRUPT, store.readCandidate().state()); + + ManagedFileIo.Reader unreadable = path -> { + throw new IOException("injected unreadable file"); + }; + FileManagedApplicationConfigStore unreadableStore = new FileManagedApplicationConfigStore( + installationRoot, new NioManagedFilePublisher(), unreadable); + assertEquals(CandidateState.UNREADABLE, unreadableStore.readCandidate().state()); + } + + @Test + void rejectsManagedWarehouseFlagsThatContradictTheSupportedStoragePolicy() throws Exception { + FileManagedApplicationConfigStore store = new FileManagedApplicationConfigStore(installationRoot); + Path candidate = installationRoot.resolve("data/config/managed-application.yml.candidate"); + store.stageCandidate(configuration("valid"), GENERATION); + ManagedDocumentCodec.Integrity.VerifiedBody document = ManagedDocumentCodec.Integrity.extract( + Files.readAllBytes(candidate)); + String unsupported = document.content() + .replace("warehouse.store.duckdb.enabled: 'false'", + "warehouse.store.duckdb.enabled: 'true'"); + Files.write(candidate, ManagedDocumentCodec.Integrity.envelope(unsupported, document.generation())); + + assertEquals(CandidateState.CORRUPT, store.readCandidate().state()); + } + + @Test + void failedPublicationLeavesTheExistingSnapshotUntouched() throws Exception { + FileManagedApplicationConfigStore initial = new FileManagedApplicationConfigStore(installationRoot); + ManagedApplicationConfig knownGood = configuration("known-good"); + initial.stageCandidate(knownGood, GENERATION); + Path candidatePath = installationRoot.resolve("data/config/managed-application.yml.candidate"); + byte[] before = Files.readAllBytes(candidatePath); + ManagedFileIo.Publisher failingPublisher = new ManagedFileIo.Publisher() { + @Override + public void publish(Path target, byte[] content, boolean ownerOnly) throws IOException { + throw new IOException("injected publication failure"); + } + + @Override + public void remove(Path target) throws IOException { + throw new IOException("injected removal failure"); + } + }; + FileManagedApplicationConfigStore failing = + new FileManagedApplicationConfigStore(installationRoot, failingPublisher); + + IOException failure = assertThrows(IOException.class, + () -> failing.stageCandidate(configuration("replacement"), "replacement-generation")); + + assertEquals("injected publication failure", failure.getMessage()); + assertArrayEquals(before, Files.readAllBytes(candidatePath)); + } + + @Test + void secretFileIsOwnerOnlyOnPosixFileSystems() throws Exception { + FileManagedSecretStore store = new FileManagedSecretStore(installationRoot); + store.stageCandidate(secrets(), GENERATION); + Path configDirectory = installationRoot.resolve("data/config"); + Path candidate = configDirectory.resolve("managed-secrets.properties.candidate"); + + if (Files.getFileStore(candidate).supportsFileAttributeView("posix")) { + assertOwnerOnly(candidate); + store.promoteCandidate(secrets(), GENERATION); + Path active = configDirectory.resolve("managed-secrets.properties"); + assertOwnerOnly(active); + store.stageCandidate(secrets(), "second-generation"); + store.promoteCandidate(secrets(), "second-generation"); + Path lastKnownGood = configDirectory.resolve("managed-secrets.properties.last-known-good"); + assertOwnerOnly(active); + assertOwnerOnly(lastKnownGood); + assertOwnerOnly(active); + } + } + + @Test + void derivesManagedPathsFromTheNormalizedAbsoluteInstallationRoot() throws Exception { + Path relativeRoot = Path.of("").toAbsolutePath().relativize(installationRoot.toAbsolutePath()); + FileManagedApplicationConfigStore store = new FileManagedApplicationConfigStore(relativeRoot); + + store.stageCandidate(configuration("relative-root"), GENERATION); + + assertTrue(Files.isRegularFile(installationRoot.resolve( + "data/config/managed-application.yml.candidate"))); + assertEquals(configuration("relative-root"), store.readCandidate().value().orElseThrow()); + } + + private static void assertOwnerOnly(Path path) throws IOException { + assertEquals(Set.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE), + Files.getPosixFilePermissions(path)); + } + + private static ManagedApplicationConfig configuration(String name) { + return new ManagedApplicationConfig( + new MetadataDatabaseSettings( + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db/" + name, "hertzbeat"), + GreptimeSettings.anonymous( + new GreptimeEndpoints("greptime:4001", "http://greptime:4000"), "public")); + } + + private static ManagedSecrets secrets() { + return ManagedSecrets.withTelemetryPassword( + SecretValue.of(DATABASE_PASSWORD), SecretValue.of(TELEMETRY_PASSWORD)); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationPortContractTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationPortContractTest.java new file mode 100644 index 0000000000..b0c749add8 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationPortContractTest.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import java.io.IOException; +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.api.Test; + +class ManagedConfigurationPortContractTest { + + private static final String SECRET = "managed-secret-value"; + + @Test + void keepsApplicationAndSecretConfigurationInSeparateTypedPorts() throws Exception { + RecordingApplicationStore applicationStore = new RecordingApplicationStore(); + RecordingSecretStore secretStore = new RecordingSecretStore(); + ManagedApplicationConfig application = configuration(); + ManagedSecrets secrets = ManagedSecrets.withTelemetryPassword( + SecretValue.of(SECRET), SecretValue.of(SECRET)); + + applicationStore.stageCandidate(application, "port-contract-generation"); + secretStore.stageCandidate(secrets, "port-contract-generation"); + + assertEquals(application, applicationStore.readCandidate().value().orElseThrow()); + assertEquals(secrets, secretStore.readCandidate().value().orElseThrow()); + } + + @Test + void secretValuesNeverRenderTheirContent() { + SecretValue password = SecretValue.of(SECRET); + ManagedSecrets secrets = ManagedSecrets.withTelemetryPassword(password, password); + + assertFalse(password.toString().contains(SECRET)); + assertFalse(secrets.toString().contains(SECRET)); + } + + @Test + void candidateReadClassifiesRecoveryWithoutExceptionsOrRawDocuments() { + ManagedApplicationConfig config = configuration(); + + assertEquals(config, CandidateRead.valid(config, "contract-generation").value().orElseThrow()); + assertEquals(CandidateState.VALID, CandidateRead.valid(config, "contract-generation").state()); + assertEquals(Optional.empty(), CandidateRead.corrupt().value()); + assertEquals(Optional.empty(), CandidateRead.invalid().value()); + assertEquals(Optional.empty(), CandidateRead.unreadable().value()); + assertEquals(CandidateState.MISSING, CandidateRead.missing().state()); + } + + private static ManagedApplicationConfig configuration() { + return new ManagedApplicationConfig( + new MetadataDatabaseSettings( + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db/hertzbeat", "hertzbeat"), + GreptimeSettings.authenticated( + new GreptimeEndpoints("greptime:4001", "http://greptime:4000"), "public", "hertzbeat")); + } + + private static final class RecordingApplicationStore implements ManagedApplicationConfigStore { + + private ManagedApplicationConfig candidate; + private String generation; + + @Override + public CandidateRead readCandidate() { + return candidate == null ? CandidateRead.missing() : CandidateRead.valid(candidate, generation); + } + + @Override + public CandidateRead readLastKnownGood() { + return CandidateRead.missing(); + } + + @Override + public CandidateRead readActive() { + return CandidateRead.missing(); + } + + @Override + public void stageCandidate(ManagedApplicationConfig candidate, String generation) { + this.candidate = candidate; + this.generation = generation; + } + + @Override + public void promoteCandidate(ManagedApplicationConfig expected, String generation) { + } + + @Override + public void restoreActive(CandidateRead previous) { + candidate = previous.value().orElse(null); + generation = previous.generation().orElse(null); + } + + @Override + public void discardCandidate() { + candidate = null; + generation = null; + } + } + + private static final class RecordingSecretStore implements ManagedSecretStore { + + private ManagedSecrets candidate; + private String generation; + + @Override + public CandidateRead readCandidate() { + return candidate == null ? CandidateRead.missing() : CandidateRead.valid(candidate, generation); + } + + @Override + public CandidateRead readLastKnownGood() { + return CandidateRead.missing(); + } + + @Override + public CandidateRead readActive() { + return CandidateRead.missing(); + } + + @Override + public void stageCandidate(ManagedSecrets candidate, String generation) throws IOException { + this.candidate = candidate; + this.generation = generation; + } + + @Override + public void promoteCandidate(ManagedSecrets expected, String generation) { + } + + @Override + public void restoreActive(CandidateRead previous) { + candidate = previous.value().orElse(null); + generation = previous.generation().orElse(null); + } + + @Override + public void discardCandidate() { + candidate = null; + generation = null; + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java new file mode 100644 index 0000000000..15e3038a6d --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java @@ -0,0 +1,316 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class ManagedConfigurationTransactionTest { + + @TempDir + private Path installationRoot; + + @Test + void applyPublishesOnlyTheValidatedAggregate() throws Exception { + ManagedConfigurationTransaction transaction = new ManagedConfigurationTransaction(installationRoot); + + assertEquals(ManagedConfigurationTransaction.Outcome.APPLIED, + transaction.apply(bundle("first"))); + + assertActivePair("first"); + } + + @Test + void rollsBackBothFilesWhenTheSecondPromotionFails() throws Exception { + assertEquals(ManagedConfigurationTransaction.Outcome.APPLIED, + new ManagedConfigurationTransaction(installationRoot).apply(bundle("previous"))); + FileManagedApplicationConfigStore applicationStore = new FileManagedApplicationConfigStore(installationRoot); + FileManagedSecretStore failingSecrets = new FileManagedSecretStore( + installationRoot, new FailingOnceActivePublicationPublisher(new NioManagedFilePublisher())); + ManagedConfigurationTransaction transaction = new ManagedConfigurationTransaction( + applicationStore, failingSecrets, installationRoot); + + assertEquals(ManagedConfigurationTransaction.Outcome.ROLLED_BACK, + transaction.apply(bundle("next"))); + assertActivePair("previous"); + } + + @Test + void filesystemWithoutAtomicMoveFailsBeforeStagingAndPreservesTheActivePair() throws Exception { + assertEquals(ManagedConfigurationTransaction.Outcome.APPLIED, + new ManagedConfigurationTransaction(installationRoot).apply(bundle("previous"))); + AtomicUnsupportedPublisher unsupported = new AtomicUnsupportedPublisher(); + FileManagedApplicationConfigStore applications = + new FileManagedApplicationConfigStore(installationRoot, unsupported); + FileManagedSecretStore secrets = new FileManagedSecretStore(installationRoot, unsupported); + + assertThrows(AtomicMoveNotSupportedException.class, + () -> new ManagedConfigurationTransaction(applications, secrets, installationRoot) + .apply(bundle("next"))); + + assertActivePair("previous"); + assertEquals(CandidateState.MISSING, applications.readCandidate().state()); + assertEquals(CandidateState.MISSING, secrets.readCandidate().state()); + } + + @ParameterizedTest + @MethodSource("interruptedPublicationStates") + void recoverConvergesEveryExplicitInterruptedPair( + CrashState crashState, ManagedConfigurationTransaction.Outcome expected, String activeSuffix) + throws Exception { + FileManagedApplicationConfigStore applications = new FileManagedApplicationConfigStore(installationRoot); + FileManagedSecretStore secrets = new FileManagedSecretStore(installationRoot); + new ManagedConfigurationTransaction(installationRoot).apply(bundle("previous")); + crashState.create(applications, secrets); + + assertEquals(expected, new ManagedConfigurationTransaction(installationRoot).recover()); + assertActivePair(activeSuffix); + if (crashState != CrashState.BEFORE_FIRST_ACTIVE_REPLACE) { + assertLastKnownGoodPair("previous"); + } + } + + @Test + void recoverFailsClosedWhenNoCompleteGenerationPairExists() throws Exception { + FileManagedApplicationConfigStore applications = new FileManagedApplicationConfigStore(installationRoot); + FileManagedSecretStore secrets = new FileManagedSecretStore(installationRoot); + applications.stageCandidate(configuration("application"), "application-generation"); + secrets.stageCandidate(secrets("secret"), "secret-generation"); + applications.promoteCandidate(configuration("application"), "application-generation"); + secrets.promoteCandidate(secrets("secret"), "secret-generation"); + + assertEquals(ManagedConfigurationTransaction.Outcome.RECOVERY_REQUIRED, + new ManagedConfigurationTransaction(installationRoot).recover()); + } + + @Test + void recoverDoesNotPromoteCandidatePairOverMismatchedActiveFiles() throws Exception { + FileManagedApplicationConfigStore applications = new FileManagedApplicationConfigStore(installationRoot); + FileManagedSecretStore secrets = new FileManagedSecretStore(installationRoot); + applications.stageCandidate(configuration("application"), "application-generation"); + applications.promoteCandidate(configuration("application"), "application-generation"); + secrets.stageCandidate(secrets("secret"), "secret-generation"); + secrets.promoteCandidate(secrets("secret"), "secret-generation"); + applications.stageCandidate(configuration("candidate"), "candidate-generation"); + secrets.stageCandidate(secrets("candidate"), "candidate-generation"); + + assertEquals(ManagedConfigurationTransaction.Outcome.RECOVERY_REQUIRED, + new ManagedConfigurationTransaction(installationRoot).recover()); + } + + @Test + void secondTransactionCannotEnterWhileSameProcessHoldsTheOsLock() throws Exception { + Path config = Files.createDirectories(installationRoot.resolve("data/config")); + Path lockPath = config.resolve(".managed-config.lock"); + try (FileChannel channel = FileChannel.open(lockPath, + StandardOpenOption.CREATE, StandardOpenOption.WRITE); + FileLock ignored = channel.lock()) { + assertThrows(IOException.class, + () -> new ManagedConfigurationTransaction(installationRoot).apply(bundle("blocked"))); + } + } + + @Test + void transactionCannotEnterWhileAnotherProcessHoldsTheOsLock() throws Exception { + Path config = Files.createDirectories(installationRoot.resolve("data/config")); + Path lockPath = config.resolve(".managed-config.lock"); + Path ready = installationRoot.resolve("lock-ready"); + Path release = installationRoot.resolve("lock-release"); + Process holder = new ProcessBuilder( + Path.of(System.getProperty("java.home"), "bin", "java").toString(), + "-cp", System.getProperty("java.class.path"), LockHolder.class.getName(), + lockPath.toString(), ready.toString(), release.toString()) + .redirectErrorStream(true) + .start(); + try { + waitForFile(ready); + assertThrows(IOException.class, + () -> new ManagedConfigurationTransaction(installationRoot).apply(bundle("blocked"))); + } finally { + Files.writeString(release, "release"); + if (!holder.waitFor(5, TimeUnit.SECONDS)) { + holder.destroyForcibly(); + holder.waitFor(5, TimeUnit.SECONDS); + } + } + assertEquals(0, holder.exitValue()); + } + + private static void waitForFile(Path ready) throws Exception { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (!Files.exists(ready) && System.nanoTime() < deadline) { + Thread.sleep(10); + } + if (!Files.exists(ready)) { + throw new AssertionError("Lock holder did not start"); + } + } + + private static Stream interruptedPublicationStates() { + return Stream.of( + Arguments.of(CrashState.BEFORE_FIRST_ACTIVE_REPLACE, + ManagedConfigurationTransaction.Outcome.ROLLED_BACK, "previous"), + Arguments.of(CrashState.BETWEEN_ACTIVE_REPLACES, + ManagedConfigurationTransaction.Outcome.APPLIED, "next"), + Arguments.of(CrashState.AFTER_BOTH_ACTIVE_REPLACES, + ManagedConfigurationTransaction.Outcome.APPLIED, "next")); + } + + private void assertActivePair(String suffix) { + assertEquals(configuration(suffix), new FileManagedApplicationConfigStore(installationRoot) + .readActive().value().orElseThrow()); + assertEquals(secrets(suffix), new FileManagedSecretStore(installationRoot) + .readActive().value().orElseThrow()); + } + + private void assertLastKnownGoodPair(String suffix) { + assertEquals(configuration(suffix), new FileManagedApplicationConfigStore(installationRoot) + .readLastKnownGood().value().orElseThrow()); + assertEquals(secrets(suffix), new FileManagedSecretStore(installationRoot) + .readLastKnownGood().value().orElseThrow()); + } + + private enum CrashState { + BEFORE_FIRST_ACTIVE_REPLACE { + @Override + void create(FileManagedApplicationConfigStore applications, FileManagedSecretStore secrets) + throws Exception { + stageNext(applications, secrets); + } + }, + BETWEEN_ACTIVE_REPLACES { + @Override + void create(FileManagedApplicationConfigStore applications, FileManagedSecretStore secrets) + throws Exception { + stageNext(applications, secrets); + applications.promoteCandidate(configuration("next"), "next-generation"); + } + }, + AFTER_BOTH_ACTIVE_REPLACES { + @Override + void create(FileManagedApplicationConfigStore applications, FileManagedSecretStore secrets) + throws Exception { + stageNext(applications, secrets); + applications.promoteCandidate(configuration("next"), "next-generation"); + secrets.promoteCandidate( + ManagedConfigurationTransactionTest.secrets("next"), "next-generation"); + } + }; + + abstract void create(FileManagedApplicationConfigStore applications, FileManagedSecretStore secrets) + throws Exception; + + static void stageNext(FileManagedApplicationConfigStore applications, FileManagedSecretStore secrets) + throws Exception { + applications.stageCandidate(configuration("next"), "next-generation"); + secrets.stageCandidate(ManagedConfigurationTransactionTest.secrets("next"), "next-generation"); + } + } + + private static ManagedConfigurationBundle bundle(String suffix) { + return new ManagedConfigurationBundle(configuration(suffix), secrets(suffix)); + } + + private static ManagedApplicationConfig configuration(String name) { + return new ManagedApplicationConfig( + new MetadataDatabaseSettings( + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db/" + name, "hertzbeat"), + GreptimeSettings.anonymous( + new GreptimeEndpoints("greptime:4001", "http://greptime:4000"), "public")); + } + + private static ManagedSecrets secrets(String suffix) { + return ManagedSecrets.withoutTelemetryPassword(SecretValue.of("database-" + suffix)); + } + + private static final class FailingOnceActivePublicationPublisher implements ManagedFileIo.Publisher { + + private final ManagedFileIo.Publisher delegate; + private boolean failNextActivePublication = true; + + private FailingOnceActivePublicationPublisher(ManagedFileIo.Publisher delegate) { + this.delegate = delegate; + } + + @Override + public void publish(Path target, byte[] content, boolean ownerOnly) throws IOException { + if (failNextActivePublication + && target.getFileName().toString().equals("managed-secrets.properties")) { + failNextActivePublication = false; + throw new IOException("injected second-file promotion failure"); + } + delegate.publish(target, content, ownerOnly); + } + + @Override + public void remove(Path target) throws IOException { + delegate.remove(target); + } + } + + private static final class AtomicUnsupportedPublisher implements ManagedFileIo.Publisher { + + @Override + public void publish(Path target, byte[] content, boolean ownerOnly) throws IOException { + throw new AtomicMoveNotSupportedException( + target.toString(), target.toString(), "injected unsupported atomic move"); + } + + @Override + public void remove(Path target) throws IOException { + Files.deleteIfExists(target); + } + } + + /** Separate JVM entry point proving that the lock coordinates processes, not only instances. */ + public static final class LockHolder { + + private LockHolder() { + } + + public static void main(String[] arguments) throws Exception { + Path lockPath = Path.of(arguments[0]); + Path ready = Path.of(arguments[1]); + Path release = Path.of(arguments[2]); + try (FileChannel channel = FileChannel.open(lockPath, + StandardOpenOption.CREATE, StandardOpenOption.WRITE); + FileLock ignored = channel.lock()) { + Files.writeString(ready, "ready"); + while (!Files.exists(release)) { + Thread.sleep(10); + } + } + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedDeploymentCapabilityTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedDeploymentCapabilityTest.java new file mode 100644 index 0000000000..aa61d42105 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedDeploymentCapabilityTest.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class ManagedDeploymentCapabilityTest { + + @TempDir + private Path installationRoot; + + @Test + void selectsManagedWriteOnlyWhenTheConfigurationLocationIsWritable() { + ManagedConfigDeploymentDetector writableDetector = + new ManagedConfigDeploymentDetector(installationRoot, path -> true); + ManagedConfigDeploymentDetector readOnlyDetector = + new ManagedConfigDeploymentDetector(installationRoot, path -> false); + + ManagedConfigCapability writable = writableDetector.detect(); + ManagedConfigCapability readOnly = readOnlyDetector.detect(); + + assertEquals(ApplyMode.MANAGED_WRITE, writable.applyMode()); + assertTrue(writable.writableManagedConfig()); + assertEquals(DeploymentConstraint.NONE, writable.constraint()); + assertEquals(ApplyMode.EXTERNAL_APPLY, readOnly.applyMode()); + assertFalse(readOnly.writableManagedConfig()); + assertEquals(DeploymentConstraint.READ_ONLY, readOnly.constraint()); + } + + @Test + void detectsFirstInstallFromWritableParent() { + ManagedConfigCapability capability = new ManagedConfigDeploymentDetector(installationRoot).detect(); + + assertEquals(ApplyMode.MANAGED_WRITE, capability.applyMode()); + assertTrue(capability.writableManagedConfig()); + } + + @Test + void distinguishesMissingAndNonDirectoryInstallationRoots() throws Exception { + Path missing = installationRoot.resolve("missing"); + Path regularFile = installationRoot.resolve("regular-file"); + java.nio.file.Files.writeString(regularFile, "not a directory"); + + assertEquals(DeploymentConstraint.INSTALLATION_ROOT_MISSING, + new ManagedConfigDeploymentDetector(missing).detect().constraint()); + assertEquals(DeploymentConstraint.INSTALLATION_ROOT_NOT_DIRECTORY, + new ManagedConfigDeploymentDetector(regularFile).detect().constraint()); + } + + @Test + void rejectsManagedConfigSymlinkEscapeInDetectorAndStore() throws Exception { + Path outside = installationRoot.resolveSibling(installationRoot.getFileName() + "-outside"); + Files.createDirectories(outside); + Path dataLink = installationRoot.resolve("data"); + try { + Files.createSymbolicLink(dataLink, outside); + } catch (UnsupportedOperationException | IOException exception) { + Assumptions.abort("Symbolic links are unavailable"); + } + + assertEquals(DeploymentConstraint.UNSAFE_PATH, + new ManagedConfigDeploymentDetector(installationRoot).detect().constraint()); + org.junit.jupiter.api.Assertions.assertThrows(IOException.class, + () -> new FileManagedApplicationConfigStore(installationRoot).stageCandidate( + new ManagedApplicationConfig( + new MetadataDatabaseSettings( + org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind.H2, + "jdbc:h2:./data/test", "sa"), + GreptimeSettings.anonymous( + new GreptimeEndpoints("localhost:4001", "http://localhost:4000"), "public")), + "symlink-test-generation")); + } + + @Test + void rejectsSymlinkedInstallationRootInDetectorAndStore() throws Exception { + Path rootLink = installationRoot.resolveSibling(installationRoot.getFileName() + "-link"); + try { + Files.createSymbolicLink(rootLink, installationRoot); + } catch (UnsupportedOperationException | IOException exception) { + Assumptions.abort("Symbolic links are unavailable"); + } + try { + assertEquals(DeploymentConstraint.UNSAFE_PATH, + new ManagedConfigDeploymentDetector(rootLink).detect().constraint()); + org.junit.jupiter.api.Assertions.assertThrows(IOException.class, + () -> new FileManagedApplicationConfigStore(rootLink).stageCandidate( + new ManagedApplicationConfig( + new MetadataDatabaseSettings( + org.apache.hertzbeat.manager.setup.api.SetupApiContract + .MetadataDatabaseKind.H2, + "jdbc:h2:./data/test", "sa"), + GreptimeSettings.anonymous( + new GreptimeEndpoints( + "localhost:4001", "http://localhost:4000"), "public")), + "root-symlink-generation")); + } finally { + Files.deleteIfExists(rootLink); + } + } + + @Test + void rejectsSymlinkedLockAndNonRegularSnapshotArtifacts() throws Exception { + Path config = Files.createDirectories(installationRoot.resolve("data/config")); + Path outside = Files.writeString(installationRoot.resolve("outside-lock"), "lock"); + Path lock = config.resolve(".managed-config.lock"); + try { + Files.createSymbolicLink(lock, outside); + } catch (UnsupportedOperationException | IOException exception) { + Assumptions.abort("Symbolic links are unavailable"); + } + assertEquals(DeploymentConstraint.UNSAFE_PATH, + new ManagedConfigDeploymentDetector(installationRoot).detect().constraint()); + + Files.delete(lock); + Files.createDirectory(config.resolve("managed-application.yml.candidate")); + assertEquals(DeploymentConstraint.UNSAFE_PATH, + new ManagedConfigDeploymentDetector(installationRoot).detect().constraint()); + } + + @Test + void checksLockCandidateAndLastKnownGoodReadWriteAccess() throws Exception { + Path config = Files.createDirectories(installationRoot.resolve("data/config")); + Path lock = Files.writeString(config.resolve(".managed-config.lock"), "lock"); + Path candidate = Files.writeString( + config.resolve("managed-application.yml.candidate"), "candidate"); + Path lastKnownGood = Files.writeString( + config.resolve("managed-secrets.properties.last-known-good"), "last-known-good"); + + assertEquals(DeploymentConstraint.READ_ONLY, + new ManagedConfigDeploymentDetector( + installationRoot, path -> !path.equals(lock), path -> true).detect().constraint()); + assertEquals(DeploymentConstraint.READ_ONLY, + new ManagedConfigDeploymentDetector( + installationRoot, path -> !path.equals(candidate), path -> true).detect().constraint()); + assertEquals(DeploymentConstraint.READ_ONLY, + new ManagedConfigDeploymentDetector( + installationRoot, path -> true, path -> !path.equals(lastKnownGood)).detect().constraint()); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/NioManagedFilePublisherTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/NioManagedFilePublisherTest.java new file mode 100644 index 0000000000..333c5491df --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/NioManagedFilePublisherTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class NioManagedFilePublisherTest { + + @TempDir + private Path temporaryDirectory; + + @Test + void failsClosedWhenAtomicMoveIsUnsupportedAndPreservesTheActiveFile() throws Exception { + AtomicInteger atomicMoves = new AtomicInteger(); + AtomicInteger directoryForces = new AtomicInteger(); + ManagedFileIo.Operations operations = new ManagedFileIo.Operations() { + @Override + public void atomicReplace(Path source, Path target) throws IOException { + atomicMoves.incrementAndGet(); + throw new AtomicMoveNotSupportedException(source.toString(), target.toString(), "injected"); + } + + @Override + public void forceDirectory(Path directory) { + directoryForces.incrementAndGet(); + } + }; + NioManagedFilePublisher publisher = new NioManagedFilePublisher(operations); + Path target = temporaryDirectory.resolve("managed.yml"); + Files.writeString(target, "active", StandardCharsets.UTF_8); + + assertThrows(AtomicMoveNotSupportedException.class, + () -> publisher.publish(target, "candidate".getBytes(StandardCharsets.UTF_8), false)); + + assertEquals(1, atomicMoves.get()); + assertEquals(0, directoryForces.get()); + assertEquals("active", Files.readString(target, StandardCharsets.UTF_8)); + } + + @Test + void doesNotHideRealAtomicMoveFailuresOrForceDirectoryAfterFailure() { + AtomicInteger directoryForces = new AtomicInteger(); + ManagedFileIo.Operations operations = new ManagedFileIo.Operations() { + @Override + public void atomicReplace(Path source, Path target) throws IOException { + throw new IOException("permission denied"); + } + + @Override + public void forceDirectory(Path directory) { + directoryForces.incrementAndGet(); + } + }; + NioManagedFilePublisher publisher = new NioManagedFilePublisher(operations); + + IOException failure = assertThrows(IOException.class, () -> publisher.publish( + temporaryDirectory.resolve("managed.yml"), "content".getBytes(StandardCharsets.UTF_8), false)); + + assertEquals("permission denied", failure.getMessage()); + assertEquals(0, directoryForces.get()); + } +} diff --git a/hertzbeat-startup/pom.xml b/hertzbeat-startup/pom.xml index e23e60b591..e2e6b5deb6 100644 --- a/hertzbeat-startup/pom.xml +++ b/hertzbeat-startup/pom.xml @@ -190,6 +190,7 @@ define/** db/** templates/** + META-INF/** **/*.html diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/config/ManagedConfigEnvironmentPostProcessor.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/config/ManagedConfigEnvironmentPostProcessor.java new file mode 100644 index 0000000000..3bc6efdb42 --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/config/ManagedConfigEnvironmentPostProcessor.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup.config; + +import java.nio.file.Files; +import java.nio.file.Path; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector; +import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector.Inspection; +import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector.State; +import org.springframework.boot.EnvironmentPostProcessor; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.context.config.ConfigDataEnvironmentPostProcessor; +import org.springframework.boot.origin.Origin; +import org.springframework.boot.origin.OriginLookup; +import org.springframework.boot.origin.OriginTrackedResource; +import org.springframework.boot.origin.TextResourceOrigin; +import org.springframework.core.Ordered; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.core.env.EnumerablePropertySource; +import org.springframework.core.env.MapPropertySource; +import org.springframework.core.env.MutablePropertySources; +import org.springframework.core.env.PropertySource; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; + +/** Loads the two fixed managed files between operator files and classpath defaults for every profile. */ +public final class ManagedConfigEnvironmentPostProcessor implements EnvironmentPostProcessor, Ordered { + + static final String INSTALLATION_ROOT_PROPERTY = "hertzbeat.internal.installation-root"; + public static final String INTERNAL_RUNTIME_PROPERTY_SOURCE = "hertzbeatInternalRuntimeMode"; + private static final String DEFAULT_INSTALLATION_ROOT = "."; + + @Override + public int getOrder() { + return ConfigDataEnvironmentPostProcessor.ORDER + 1; + } + + @Override + public void postProcessEnvironment(ConfigurableEnvironment environment, SpringApplication application) { + PropertySource internalMode = environment.getPropertySources().get(INTERNAL_RUNTIME_PROPERTY_SOURCE); + RuntimeMode mode = internalMode == null ? RuntimeMode.NORMAL + : RuntimeMode.fromProperty((String) internalMode.getProperty(RuntimeMode.PROPERTY_NAME)); + if (internalMode != null) { + environment.getPropertySources().remove(INTERNAL_RUNTIME_PROPERTY_SOURCE); + environment.getPropertySources().addFirst(internalMode); + } + if (mode == RuntimeMode.SETUP_ONLY || mode == RuntimeMode.RECOVERY) { + return; + } + Path installationRoot = Path.of(environment.getProperty( + INSTALLATION_ROOT_PROPERTY, DEFAULT_INSTALLATION_ROOT)).toAbsolutePath().normalize(); + Path directory = installationRoot.resolve("data/config"); + if (Files.isSymbolicLink(installationRoot) + || Files.isSymbolicLink(directory.getParent()) || Files.isSymbolicLink(directory)) { + throw recoveryRequired(); + } + Inspection inspection = new ManagedActiveConfigurationInspector(installationRoot).inspect(); + if (inspection.state() == State.ABSENT) { + return; + } + if (inspection.state() != State.LOADABLE) { + throw recoveryRequired(); + } + addBetweenExternalAndClasspath(environment.getPropertySources(), new MapPropertySource( + ManagedActiveConfigurationInspector.MANAGED_APPLICATION_SOURCE, + inspection.applicationProperties())); + addBetweenExternalAndClasspath(environment.getPropertySources(), new MapPropertySource( + ManagedActiveConfigurationInspector.MANAGED_SECRET_SOURCE, + inspection.secretProperties())); + } + + private static void addBetweenExternalAndClasspath( + MutablePropertySources propertySources, PropertySource managed) { + for (PropertySource source : propertySources) { + if (isClasspathConfigData(source)) { + propertySources.addBefore(source.getName(), managed); + return; + } + } + propertySources.addLast(managed); + } + + private static boolean isClasspathConfigData(PropertySource source) { + if (!(source instanceof EnumerablePropertySource enumerable)) { + return false; + } + for (String propertyName : enumerable.getPropertyNames()) { + Origin origin = OriginLookup.getOrigin(source, propertyName); + if (origin instanceof TextResourceOrigin textOrigin + && unwrap(textOrigin.getResource()) instanceof ClassPathResource) { + return true; + } + } + return false; + } + + private static Resource unwrap(Resource resource) { + Resource current = resource; + while (current instanceof OriginTrackedResource tracked) { + current = tracked.getResource(); + } + return current; + } + + private static IllegalStateException recoveryRequired() { + return new IllegalStateException("Managed configuration requires recovery"); + } +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/SpringStartupContextLauncher.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/SpringStartupContextLauncher.java index 9aeaf6c270..e0f013ff87 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/SpringStartupContextLauncher.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/SpringStartupContextLauncher.java @@ -22,9 +22,11 @@ import org.apache.hertzbeat.bootstrap.SetupOnlyApplication; import org.apache.hertzbeat.common.runtime.RuntimeMode; import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; import org.apache.hertzbeat.startup.HertzBeatApplication; +import org.apache.hertzbeat.startup.config.ManagedConfigEnvironmentPostProcessor; import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.core.env.MapPropertySource; +import org.springframework.core.env.StandardEnvironment; /** Spring implementation with explicit AOT-visible source classes. */ public final class SpringStartupContextLauncher implements StartupContextLauncher { @@ -38,11 +40,13 @@ public final class SpringStartupContextLauncher implements StartupContextLaunche ConfigurableApplicationContext launchSpringContext( StartupDecision decision, String[] args, SetupRuntimeTransition setupRuntimeTransition) { + StandardEnvironment environment = new StandardEnvironment(); + environment.getPropertySources().addFirst(new MapPropertySource( + ManagedConfigEnvironmentPostProcessor.INTERNAL_RUNTIME_PROPERTY_SOURCE, + Map.of(RuntimeMode.PROPERTY_NAME, decision.mode().value()))); return new SpringApplicationBuilder(sourceFor(decision.mode())) + .environment(environment) .initializers(context -> { - context.getEnvironment().getPropertySources().addFirst( - new MapPropertySource("hertzbeatInternalRuntimeMode", - Map.of(RuntimeMode.PROPERTY_NAME, decision.mode().value()))); context.getBeanFactory().registerSingleton( "setupRuntimeTransition", setupRuntimeTransition); }) diff --git a/hertzbeat-startup/src/main/resources/META-INF/spring.factories b/hertzbeat-startup/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..34d3fd25a3 --- /dev/null +++ b/hertzbeat-startup/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.EnvironmentPostProcessor=\ +org.apache.hertzbeat.startup.config.ManagedConfigEnvironmentPostProcessor diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/config/ManagedConfigDataPrecedenceTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/config/ManagedConfigDataPrecedenceTest.java new file mode 100644 index 0000000000..65aa60f5bb --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/config/ManagedConfigDataPrecedenceTest.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.Map; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigSource; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.config.EffectiveConfigurationResolver; +import org.apache.hertzbeat.manager.setup.config.EffectiveConfigurationValue; +import org.apache.hertzbeat.manager.setup.config.GreptimeEndpoints; +import org.apache.hertzbeat.manager.setup.config.GreptimeSettings; +import org.apache.hertzbeat.manager.setup.config.ManagedApplicationConfig; +import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigurationBundle; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.ManagedSecrets; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.RestartRequirement; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.WebApplicationType; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.core.env.MapPropertySource; +import org.springframework.core.env.PropertySource; +import org.springframework.core.env.StandardEnvironment; +import org.springframework.core.env.SystemEnvironmentPropertySource; + +class ManagedConfigDataPrecedenceTest { + + private static final String KEY = "spring.datasource.username"; + private static final String SECRET_KEY = "spring.datasource.password"; + private static final String INSTALLATION_ROOT = "hertzbeat.internal.installation-root"; + private static final String TEST_PASSWORD = + " \\=:#!\t\r\n\fGrüße-${UNRESOLVED_TEST_SECRET}-\\${ESCAPED_TEST_SECRET} "; + + @TempDir + private Path temporaryDirectory; + + @Test + void realConfigDataHonorsEverySupportedPrecedenceLayerAndOrigin() throws Exception { + Path installationRoot = Files.createDirectories(temporaryDirectory.resolve("installation")); + ManagedSecrets managedSecrets = ManagedSecrets.withoutTelemetryPassword(SecretValue.of(TEST_PASSWORD)); + ManagedConfigurationTransaction transaction = new ManagedConfigurationTransaction(installationRoot); + assertEquals(ManagedConfigurationTransaction.Outcome.APPLIED, + transaction.apply(new ManagedConfigurationBundle(managedApplication(), managedSecrets))); + Path emptyInstallationRoot = Files.createDirectories(temporaryDirectory.resolve("empty-installation")); + Path externalDirectory = Files.createDirectories(temporaryDirectory.resolve("external")); + Files.writeString(externalDirectory.resolve("application.yml"), yaml("external")); + + assertLayer(Map.of(INSTALLATION_ROOT, emptyInstallationRoot.toString()), Map.of(), + new String[0], "sa", "class path resource", ConfigSource.BUILT_IN_DEFAULT); + assertLayer(Map.of(INSTALLATION_ROOT, installationRoot.toString()), Map.of(), + new String[0], "managed", ManagedActiveConfigurationInspector.MANAGED_APPLICATION_SOURCE, + ConfigSource.UI_MANAGED); + assertLayer(Map.of(INSTALLATION_ROOT, installationRoot.toString()), Map.of(), + new String[] {"--spring.profiles.active=managed-proof"}, + "managed", ManagedActiveConfigurationInspector.MANAGED_APPLICATION_SOURCE, + ConfigSource.UI_MANAGED); + assertLayer(Map.of( + INSTALLATION_ROOT, installationRoot.toString(), + "spring.config.additional-location", externalDirectory.toUri().toString()), + Map.of(), new String[0], "external", "application.yml", ConfigSource.EXTERNAL_FILE); + assertLayer(Map.of(INSTALLATION_ROOT, installationRoot.toString()), + Map.of("SPRING_DATASOURCE_USERNAME", "environment"), + new String[0], "environment", StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME, + ConfigSource.ENVIRONMENT); + assertLayer(Map.of(INSTALLATION_ROOT, installationRoot.toString(), KEY, "system"), + Map.of("SPRING_DATASOURCE_USERNAME", "environment"), + new String[0], "system", StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME, + ConfigSource.SYSTEM_PROPERTY); + assertLayer(Map.of(INSTALLATION_ROOT, installationRoot.toString(), KEY, "system"), + Map.of("SPRING_DATASOURCE_USERNAME", "environment"), + new String[] {"--" + KEY + "=cli"}, "cli", "commandLineArgs", ConfigSource.COMMAND_LINE); + } + + private static void assertLayer( + Map systemProperties, + Map environmentVariables, + String[] arguments, + String expected, + String originFragment, + ConfigSource expectedSource) { + ConfigurableEnvironment environment = new StandardEnvironment(); + Map systemValues = new LinkedHashMap<>(systemProperties); + systemValues.putIfAbsent("PID", "managed-config-test"); + environment.getPropertySources().replace( + StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME, + new MapPropertySource(StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME, + systemValues)); + environment.getPropertySources().replace( + StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME, + new SystemEnvironmentPropertySource(StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME, + new LinkedHashMap<>(environmentVariables))); + SpringApplication application = new SpringApplication(ProbeConfiguration.class); + application.setEnvironment(environment); + application.setWebApplicationType(WebApplicationType.NONE); + application.setLogStartupInfo(false); + try (ConfigurableApplicationContext context = application.run(arguments)) { + assertEquals(expected, context.getEnvironment().getProperty(KEY)); + if (expectedSource == ConfigSource.UI_MANAGED) { + assertEquals(TEST_PASSWORD, context.getEnvironment().getProperty(SECRET_KEY)); + } + PropertySource winner = winningSource(context.getEnvironment(), KEY); + if (expectedSource == ConfigSource.UI_MANAGED) { + assertEquals(originFragment, winner.getName()); + } else { + assertTrue(winner.getName().contains(originFragment), winner.getName()); + } + EffectiveConfigurationValue resolved = new EffectiveConfigurationResolver().resolve( + context.getEnvironment(), KEY, RestartRequirement.RESTART_REQUIRED); + assertEquals(expected, resolved.value()); + assertEquals(expectedSource, resolved.source()); + assertEquals(RestartRequirement.RESTART_REQUIRED, resolved.restartRequirement()); + } + } + + private static PropertySource winningSource(ConfigurableEnvironment environment, String key) { + for (PropertySource source : environment.getPropertySources()) { + if (!"configurationProperties".equals(source.getName()) && source.getProperty(key) != null) { + return source; + } + } + throw new AssertionError("No property source for " + key); + } + + private static String yaml(String username) { + return "spring:\n datasource:\n username: " + username + "\n"; + } + + private static ManagedApplicationConfig managedApplication() { + return new ManagedApplicationConfig( + new MetadataDatabaseSettings( + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db/hertzbeat", "managed"), + GreptimeSettings.anonymous( + new GreptimeEndpoints("greptime:4001", "http://greptime:4000"), "public")); + } + + @Configuration(proxyBeanMethods = false) + static class ProbeConfiguration { + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/ManagedConfigRecoveryContextTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/ManagedConfigRecoveryContextTest.java new file mode 100644 index 0000000000..75a2b0cfe9 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/ManagedConfigRecoveryContextTest.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup.runtime; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.nio.file.Files; +import java.nio.file.Path; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.config.GreptimeEndpoints; +import org.apache.hertzbeat.manager.setup.config.GreptimeSettings; +import org.apache.hertzbeat.manager.setup.config.ManagedApplicationConfig; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigurationBundle; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.ManagedSecrets; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.springframework.context.ConfigurableApplicationContext; + +class ManagedConfigRecoveryContextTest { + + @TempDir + private Path temporaryDirectory; + + @ParameterizedTest + @EnumSource(BrokenActivePair.class) + void brokenActivePairFallsBackToRealRecoveryContext(BrokenActivePair brokenPair) throws Exception { + Path installationRoot = Files.createDirectories( + temporaryDirectory.resolve(brokenPair.name().toLowerCase(java.util.Locale.ROOT))); + brokenPair.create(installationRoot); + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( + StartupDecision::normal, new SpringStartupContextLauncher()); + + coordinator.start(new String[] { + "--hertzbeat.internal.installation-root=" + installationRoot, + "--hertzbeat.runtime.mode=normal", + "--server.port=0", + "--spring.main.banner-mode=off" + }); + + assertEquals(RuntimeMode.RECOVERY, coordinator.mode()); + coordinator.currentContext().close(); + } + + @ParameterizedTest + @EnumSource(BrokenActivePair.class) + void commandLineCannotMakeSetupOnlyLoadBrokenManagedFiles(BrokenActivePair brokenPair) throws Exception { + Path installationRoot = Files.createDirectories( + temporaryDirectory.resolve("setup-" + brokenPair.name().toLowerCase(java.util.Locale.ROOT))); + brokenPair.create(installationRoot); + StartupDecision setupOnly = new StartupDecision( + RuntimeMode.SETUP_ONLY, SetupPhase.CONFIGURATION_REQUIRED, null); + + try (ConfigurableApplicationContext context = new SpringStartupContextLauncher().launchSpringContext( + setupOnly, + new String[] { + "--hertzbeat.internal.installation-root=" + installationRoot, + "--hertzbeat.runtime.mode=normal", + "--server.port=0", + "--spring.main.banner-mode=off" + }, () -> { })) { + assertEquals(RuntimeMode.SETUP_ONLY.value(), + context.getEnvironment().getProperty(RuntimeMode.PROPERTY_NAME)); + } + } + + private enum BrokenActivePair { + CORRUPT { + @Override + void create(Path root) throws Exception { + createValidPair(root); + Files.writeString(root.resolve("data/config/managed-application.yml"), + "spring.datasource.username: tampered\n"); + } + }, + UNREADABLE { + @Override + void create(Path root) throws Exception { + createValidPair(root); + Path application = root.resolve("data/config/managed-application.yml"); + Files.delete(application); + Files.createDirectory(application); + } + }, + GENERATION_MISMATCH { + @Override + void create(Path root) throws Exception { + Path applicationPair = root.resolve("application-pair"); + Path secretPair = root.resolve("secret-pair"); + createValidPair(applicationPair); + createValidPair(secretPair); + Path config = Files.createDirectories(root.resolve("data/config")); + Files.copy(applicationPair.resolve("data/config/managed-application.yml"), + config.resolve("managed-application.yml")); + Files.copy(secretPair.resolve("data/config/managed-secrets.properties"), + config.resolve("managed-secrets.properties")); + } + }; + + abstract void create(Path root) throws Exception; + } + + private static void createValidPair(Path installationRoot) throws Exception { + ManagedConfigurationTransaction transaction = new ManagedConfigurationTransaction(installationRoot); + assertEquals(ManagedConfigurationTransaction.Outcome.APPLIED, + transaction.apply(new ManagedConfigurationBundle(configuration(), secrets()))); + } + + private static ManagedApplicationConfig configuration() { + return new ManagedApplicationConfig( + new MetadataDatabaseSettings( + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db/hertzbeat", "hertzbeat"), + GreptimeSettings.anonymous( + new GreptimeEndpoints("greptime:4001", "http://greptime:4000"), "public")); + } + + private static ManagedSecrets secrets() { + return ManagedSecrets.withoutTelemetryPassword(SecretValue.of("recovery-test-password")); + } +} From 410161c3f45a4027c6481cb370e783c4c25e50c6 Mon Sep 17 00:00:00 2001 From: Logic Date: Sat, 8 Aug 2026 12:11:36 +0800 Subject: [PATCH 10/71] Add database-backed setup identity --- .../ObservabilityAccessTokenGateway.java | 4 +- hertzbeat-manager/pom.xml | 4 + .../config/ApiTokenValidationFilter.java | 19 +- .../hertzbeat/manager/dao/AuthTokenDao.java | 7 + .../manager/service/AccountService.java | 3 + .../service/impl/AccountServiceImpl.java | 112 +++++---- .../identity/AccountCredentialVerifier.java | 86 +++++++ .../identity/AdministratorCredentials.java | 57 +++++ .../identity/BcryptPasswordProcessor.java | 58 +++++ .../identity/BootstrapIdentityConflict.java | 44 ++++ .../setup/identity/CredentialRevocation.java | 23 ++ .../setup/identity/DatabaseAccount.java | 124 ++++++++++ .../identity/DatabaseAccountRepository.java | 39 ++++ .../DatabaseFirstAccountProvider.java | 148 ++++++++++++ ...atabaseIdentityProcessorConfiguration.java | 69 ++++++ .../IdentityInitializationService.java | 79 +++++++ .../identity/IdentityPasswordPolicy.java | 37 +++ .../LegacyAccountMigrationService.java | 85 +++++++ .../setup/identity/LegacyAccountSource.java | 34 +++ .../PersistedTokenCredentialRevocation.java | 37 +++ .../setup/identity/VersionedAccount.java | 27 +++ .../setup/identity/VersionedJwtProcessor.java | 60 +++++ .../setup/installation/DatabasePresence.java | 21 ++ .../installation/InstallationClassifier.java | 42 ++++ .../InstallationCompletionService.java | 59 +++++ .../installation/InstallationFingerprint.java | 35 +++ .../setup/installation/InstallationMode.java | 21 ++ .../installation/InstallationRecord.java | 59 +++++ .../InstallationRecordRepository.java | 24 ++ .../LocalInstallationFingerprintStore.java | 59 +++++ .../setup/security/RemoteSetupUnlock.java | 221 ++++++++++++++++++ .../setup/security/SecureSetupFile.java | 70 ++++++ .../setup/security/SetupAccessCookie.java | 36 +++ .../setup/security/SetupAccessSession.java | 44 ++++ .../setup/security/SetupUnlockCode.java | 43 ++++ .../setup/security/SetupUnlockRejected.java | 47 ++++ .../manager/ui/session/UiSessionService.java | 5 +- .../config/ApiTokenValidationFilterTest.java | 24 +- .../service/AccountCredentialVersionTest.java | 143 ++++++++++++ .../manager/service/AccountServiceTest.java | 16 +- .../identity/BcryptPasswordProcessorTest.java | 88 +++++++ .../DatabaseFirstAccountProviderTest.java | 93 ++++++++ ...aseIdentityProcessorConfigurationTest.java | 46 ++++ .../DatabaseIdentitySpringWiringTest.java | 55 +++++ .../IdentityInitializationServiceTest.java | 136 +++++++++++ .../LegacyAccountMigrationServiceTest.java | 138 +++++++++++ .../identity/VersionedJwtProcessorTest.java | 82 +++++++ .../InstallationClassifierTest.java | 45 ++++ .../InstallationPersistenceTest.java | 123 ++++++++++ .../setup/security/RemoteSetupUnlockTest.java | 176 ++++++++++++++ .../ui/session/UiSessionServiceTest.java | 6 +- .../config/OtlpGrpcServerConfig.java | 3 +- .../config/OtlpGrpcServerConfigTest.java | 22 +- hertzbeat-startup/pom.xml | 22 ++ .../V205__add_identity_and_installation.sql | 20 ++ .../V205__add_identity_and_installation.sql | 20 ++ .../V205__add_identity_and_installation.sql | 20 ++ .../IdentityMigrationDatabaseTest.java | 122 ++++++++++ .../IdentityMigrationResourceTest.java | 103 ++++++++ ...ckagedLegacyDefaultAuthenticationTest.java | 44 ++++ 60 files changed, 3409 insertions(+), 80 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/AccountCredentialVerifier.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/AdministratorCredentials.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/BcryptPasswordProcessor.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/BootstrapIdentityConflict.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/CredentialRevocation.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/DatabaseAccount.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/DatabaseAccountRepository.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/DatabaseFirstAccountProvider.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/DatabaseIdentityProcessorConfiguration.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/IdentityInitializationService.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/IdentityPasswordPolicy.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/LegacyAccountMigrationService.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/LegacyAccountSource.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/PersistedTokenCredentialRevocation.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/VersionedAccount.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/VersionedJwtProcessor.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/DatabasePresence.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationClassifier.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationCompletionService.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationFingerprint.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationMode.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationRecord.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationRecordRepository.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/LocalInstallationFingerprintStore.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/RemoteSetupUnlock.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFile.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupAccessCookie.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupAccessSession.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupUnlockCode.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupUnlockRejected.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/AccountCredentialVersionTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/BcryptPasswordProcessorTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/DatabaseFirstAccountProviderTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/DatabaseIdentityProcessorConfigurationTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/DatabaseIdentitySpringWiringTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/IdentityInitializationServiceTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/LegacyAccountMigrationServiceTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/VersionedJwtProcessorTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/installation/InstallationClassifierTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/installation/InstallationPersistenceTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/RemoteSetupUnlockTest.java create mode 100644 hertzbeat-startup/src/main/resources/db/migration/h2/V205__add_identity_and_installation.sql create mode 100644 hertzbeat-startup/src/main/resources/db/migration/mysql/V205__add_identity_and_installation.sql create mode 100644 hertzbeat-startup/src/main/resources/db/migration/postgresql/V205__add_identity_and_installation.sql create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/identity/IdentityMigrationDatabaseTest.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/identity/IdentityMigrationResourceTest.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/identity/PackagedLegacyDefaultAuthenticationTest.java diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/observability/gateway/ObservabilityAccessTokenGateway.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/observability/gateway/ObservabilityAccessTokenGateway.java index a78f3bc408..14cdff56fe 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/observability/gateway/ObservabilityAccessTokenGateway.java +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/observability/gateway/ObservabilityAccessTokenGateway.java @@ -25,6 +25,7 @@ import java.util.List; public interface ObservabilityAccessTokenGateway { String CLAIM_MANAGED = "managed"; + String CLAIM_CREDENTIAL_VERSION = "credentialVersion"; /** * Check the status of a managed token. @@ -62,9 +63,10 @@ public interface ObservabilityAccessTokenGateway { * * @param userId subject from token * @param claimedRoles roles embedded in token + * @param credentialVersion account credential generation embedded in token, or null for legacy accounts * @return null when owner is still allowed, otherwise rejection reason */ - String checkManagedTokenAccess(String userId, List claimedRoles); + String checkManagedTokenAccess(String userId, List claimedRoles, Long credentialVersion); /** * Touch token last used time. diff --git a/hertzbeat-manager/pom.xml b/hertzbeat-manager/pom.xml index cb6fa668b3..549c402454 100644 --- a/hertzbeat-manager/pom.xml +++ b/hertzbeat-manager/pom.xml @@ -172,6 +172,10 @@ com.usthe.sureness spring-boot3-starter-sureness + + org.springframework.security + spring-security-crypto + com.squareup.okhttp3 diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/config/ApiTokenValidationFilter.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/config/ApiTokenValidationFilter.java index 0f07aec9b2..deecfcf614 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/config/ApiTokenValidationFilter.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/config/ApiTokenValidationFilter.java @@ -35,9 +35,9 @@ import org.apache.hertzbeat.collector.dispatch.DispatchConstants; import org.apache.hertzbeat.common.constants.NetworkConstants; import org.apache.hertzbeat.common.observability.gateway.AuthTokenRequestContext; import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes; +import org.apache.hertzbeat.common.observability.gateway.ObservabilityAccessTokenGateway; import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.manager.service.AccountService; -import org.apache.hertzbeat.manager.service.impl.AccountServiceImpl; import org.jspecify.annotations.NonNull; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Component; @@ -103,7 +103,7 @@ public class ApiTokenValidationFilter implements HandlerInterceptor { } touchTokenLastUsedTime(token); } catch (RuntimeException e) { - log.warn("Managed token validation failed"); + log.warn("Managed token validation failed ({})", e.getClass().getSimpleName()); return writeError(response, HttpStatus.SERVICE_UNAVAILABLE, TOKEN_VALIDATION_UNAVAILABLE); } } @@ -135,7 +135,8 @@ public class ApiTokenValidationFilter implements HandlerInterceptor { if (rejectReason != null) { return rejectReason; } - rejectReason = accountService.checkManagedTokenAccess(getCurrentUserId(subject), extractClaimedRoles(subject)); + rejectReason = accountService.checkManagedTokenAccess( + getCurrentUserId(subject), extractClaimedRoles(subject), extractCredentialVersion(subject)); if (rejectReason != null) { return rejectReason; } @@ -155,7 +156,7 @@ public class ApiTokenValidationFilter implements HandlerInterceptor { if (principalMap == null) { return false; } - Object managed = principalMap.getPrincipal(AccountServiceImpl.CLAIM_MANAGED); + Object managed = principalMap.getPrincipal(ObservabilityAccessTokenGateway.CLAIM_MANAGED); return managed instanceof Boolean ? (Boolean) managed : Boolean.parseBoolean(String.valueOf(managed)); } @@ -181,6 +182,16 @@ public class ApiTokenValidationFilter implements HandlerInterceptor { return principal == null ? null : String.valueOf(principal); } + private Long extractCredentialVersion(SubjectSum subject) { + PrincipalMap principalMap = subject.getPrincipalMap(); + if (principalMap == null) { + return null; + } + Object claimedVersion = principalMap.getPrincipal( + ObservabilityAccessTokenGateway.CLAIM_CREDENTIAL_VERSION); + return claimedVersion instanceof Number number ? number.longValue() : null; + } + private String bindManagedCollectorBoundary(HttpServletRequest request, SubjectSum subject) { PrincipalMap principalMap = subject.getPrincipalMap(); if (principalMap == null || !AuthTokenScopes.MANAGED_COLLECTOR_AUDIENCE.equals( diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/dao/AuthTokenDao.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/dao/AuthTokenDao.java index e30c86d11c..623e539350 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/dao/AuthTokenDao.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/dao/AuthTokenDao.java @@ -104,4 +104,11 @@ public interface AuthTokenDao extends JpaRepository, JpaSpecifi @Transactional @Query("UPDATE AuthToken t SET t.lastUsedTime = :lastUsedTime WHERE t.tokenHash = :tokenHash") void updateLastUsedTime(@Param("tokenHash") String tokenHash, @Param("lastUsedTime") LocalDateTime lastUsedTime); + + @Modifying + @Transactional + @Query("UPDATE AuthToken t SET t.status = 1, t.revokedBy = :revokedBy, t.revokedTime = :revokedTime " + + "WHERE t.creator = :creator AND t.status = 0") + int revokeActiveByCreator(@Param("creator") String creator, @Param("revokedBy") String revokedBy, + @Param("revokedTime") LocalDateTime revokedTime); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/AccountService.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/AccountService.java index 472162cff9..19c3a3f0cd 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/AccountService.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/AccountService.java @@ -113,4 +113,7 @@ public interface AccountService extends ObservabilityAccessTokenGateway { */ TokenRevocationResult deleteToken(Long id) throws AuthenticationException; + /** Validates roles and the credential generation carried by a browser session. */ + String checkSessionAccess(String userId, List claimedRoles, Long credentialVersion); + } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/AccountServiceImpl.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/AccountServiceImpl.java index a662673940..f89b37f612 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/AccountServiceImpl.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/AccountServiceImpl.java @@ -21,10 +21,8 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import com.usthe.sureness.provider.SurenessAccount; import com.usthe.sureness.provider.SurenessAccountProvider; -import com.usthe.sureness.provider.ducument.DocumentAccountProvider; import com.usthe.sureness.subject.SubjectSum; import com.usthe.sureness.util.JsonWebTokenUtil; -import com.usthe.sureness.util.Md5Util; import com.usthe.sureness.util.SurenessContextHolder; import io.jsonwebtoken.Claims; import lombok.extern.slf4j.Slf4j; @@ -33,12 +31,14 @@ import org.apache.hertzbeat.alert.util.CryptoUtils; import org.apache.hertzbeat.common.entity.manager.AuthToken; import org.apache.hertzbeat.common.observability.gateway.AuthTokenRequestContext; import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes; +import org.apache.hertzbeat.common.observability.gateway.ObservabilityAccessTokenGateway; import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.manager.dao.AuthTokenDao; import org.apache.hertzbeat.manager.pojo.dto.LoginDto; import org.apache.hertzbeat.manager.pojo.dto.RefreshTokenResponse; import org.apache.hertzbeat.manager.service.AccountService; -import org.springframework.beans.factory.annotation.Autowired; +import org.apache.hertzbeat.manager.setup.identity.AccountCredentialVerifier; +import org.apache.hertzbeat.manager.setup.identity.VersionedAccount; import org.springframework.core.Ordered; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Service; @@ -63,7 +63,6 @@ import java.util.concurrent.TimeUnit; public class AccountServiceImpl implements AccountService { private static final String REFRESH_CLAIM = "refresh"; - /** * Token validity time in seconds */ @@ -80,13 +79,6 @@ public class AccountServiceImpl implements AccountService { private static final long MAX_ACTIVE_TOKENS_PER_SCOPE_PER_USER = 20; - /** - * Custom JWT claim key to mark tokens as managed (persisted in DB for lifecycle management). - * Only tokens with this claim will be validated against the database. - * Legacy tokens without this claim are allowed to pass through for backward compatibility. - */ - public static final String CLAIM_MANAGED = "managed"; - /** * Minimum interval (in minutes) between lastUsedTime DB updates for the same token. * Reduces write pressure under high-frequency requests. @@ -112,40 +104,32 @@ public class AccountServiceImpl implements AccountService { * account data provider */ private final SurenessAccountProvider accountProvider; + private final AccountCredentialVerifier credentialVerifier; - public AccountServiceImpl() { - this(new DocumentAccountProvider()); - } - - public AccountServiceImpl(SurenessAccountProvider accountProvider) { + public AccountServiceImpl(SurenessAccountProvider accountProvider, AuthTokenDao authTokenDao, + AccountCredentialVerifier credentialVerifier) { this.accountProvider = accountProvider; + this.authTokenDao = authTokenDao; + this.credentialVerifier = credentialVerifier; } - @Autowired - private AuthTokenDao authTokenDao; + private final AuthTokenDao authTokenDao; @Override public Map authGetToken(LoginDto loginDto) throws AuthenticationException { SurenessAccount account = accountProvider.loadAccount(loginDto.getIdentifier()); - if (account == null || StringUtils.isBlank(account.getPassword())) { + if (!credentialVerifier.matches(account, loginDto.getCredential())) { throw new AuthenticationException("Incorrect Account or Password"); - } else { - String password = loginDto.getCredential(); - if (StringUtils.isNotBlank(account.getSalt())) { - password = Md5Util.md5(password + account.getSalt()); - } - if (!account.getPassword().equals(password)) { - throw new AuthenticationException("Incorrect Account or Password"); - } - if (account.isDisabledAccount() || account.isExcessiveAttempts()) { - throw new AuthenticationException("Expired or Illegal Account"); - } + } + if (!credentialVerifier.usable(account)) { + throw new AuthenticationException("Expired or Illegal Account"); } // Get the roles the user has - rbac List roles = account.getOwnRoles(); // Issue TOKEN - String issueToken = issueAccessToken(loginDto.getIdentifier(), roles, PERIOD_TIME); - String issueRefresh = issueRefreshToken(loginDto.getIdentifier(), PERIOD_TIME << 5); + Long credentialVersion = credentialVersion(account); + String issueToken = issueAccessToken(loginDto.getIdentifier(), roles, PERIOD_TIME, credentialVersion); + String issueRefresh = issueRefreshToken(loginDto.getIdentifier(), PERIOD_TIME << 5, credentialVersion); Map resp = new HashMap<>(2); resp.put("token", issueToken); resp.put("refreshToken", issueRefresh); @@ -169,9 +153,15 @@ public class AccountServiceImpl implements AccountService { if (account.isDisabledAccount() || account.isExcessiveAttempts()) { throw new AuthenticationException("Expired or Illegal Account"); } + Long tokenVersion = claims.get( + ObservabilityAccessTokenGateway.CLAIM_CREDENTIAL_VERSION, Long.class); + if (!credentialVersionMatches(account, tokenVersion)) { + throw new AuthenticationException("Expired or Illegal Account"); + } List roles = account.getOwnRoles(); - String issueToken = issueAccessToken(userId, roles, PERIOD_TIME); - String issueRefresh = issueRefreshToken(userId, PERIOD_TIME << 5); + Long credentialVersion = credentialVersion(account); + String issueToken = issueAccessToken(userId, roles, PERIOD_TIME, credentialVersion); + String issueRefresh = issueRefreshToken(userId, PERIOD_TIME << 5, credentialVersion); return new RefreshTokenResponse(issueToken, issueRefresh); } @@ -230,8 +220,9 @@ public class AccountServiceImpl implements AccountService { throw new AuthenticationException("Token quota exceeded"); } List roles = account.getOwnRoles(); + Long credentialVersion = credentialVersion(account); String token = issueApiToken(userId, roles, expireSeconds, normalizedScope, normalizedWorkspaceId, - tokenAudience, collectorId, allowedSignals); + tokenAudience, collectorId, allowedSignals, credentialVersion); // Persist token metadata for management String tokenHash = CryptoUtils.sha256Hex(token); @@ -265,10 +256,13 @@ public class AccountServiceImpl implements AccountService { } } - private String issueAccessToken(String userId, List roles, long expirationSeconds) { - Map customClaimMap = new HashMap<>(2); + private String issueAccessToken(String userId, List roles, long expirationSeconds, Long credentialVersion) { + Map customClaimMap = new HashMap<>(3); customClaimMap.put(AuthTokenScopes.CLAIM_TOKEN_SCOPE, AuthTokenScopes.UI_SESSION); customClaimMap.put(AuthTokenScopes.CLAIM_WORKSPACE_ID, AuthTokenScopes.DEFAULT_WORKSPACE_ID); + if (credentialVersion != null) { + customClaimMap.put(ObservabilityAccessTokenGateway.CLAIM_CREDENTIAL_VERSION, credentialVersion); + } return JsonWebTokenUtil.issueJwt(userId, expirationSeconds, roles, customClaimMap); } @@ -389,11 +383,27 @@ public class AccountServiceImpl implements AccountService { } @Override - public String checkManagedTokenAccess(String userId, List claimedRoles) { + public String checkManagedTokenAccess(String userId, List claimedRoles, Long credentialVersion) { + return checkCurrentAccess(userId, claimedRoles, credentialVersion); + } + + @Override + public String checkSessionAccess(String userId, List claimedRoles, Long credentialVersion) { + return checkCurrentAccess(userId, claimedRoles, credentialVersion); + } + + private String checkCurrentAccess(String userId, List claimedRoles, Long credentialVersion) { if (StringUtils.isBlank(userId)) { return "Token owner account is no longer valid"; } SurenessAccount account = accountProvider.loadAccount(userId); + if (!credentialVersionMatches(account, credentialVersion)) { + return "Token credentials are outdated"; + } + return currentAccountAccess(account, claimedRoles); + } + + private static String currentAccountAccess(SurenessAccount account, List claimedRoles) { if (account == null || account.isDisabledAccount() || account.isExcessiveAttempts()) { return "Token owner account is no longer valid"; } @@ -433,11 +443,15 @@ public class AccountServiceImpl implements AccountService { String workspaceId, String tokenAudience, String collectorId, - List allowedSignals) { - Map customClaimMap = new HashMap<>(7); - customClaimMap.put(CLAIM_MANAGED, true); + List allowedSignals, + Long credentialVersion) { + Map customClaimMap = new HashMap<>(8); + customClaimMap.put(ObservabilityAccessTokenGateway.CLAIM_MANAGED, true); customClaimMap.put(AuthTokenScopes.CLAIM_TOKEN_SCOPE, tokenScope); customClaimMap.put(AuthTokenScopes.CLAIM_WORKSPACE_ID, workspaceId); + if (credentialVersion != null) { + customClaimMap.put(ObservabilityAccessTokenGateway.CLAIM_CREDENTIAL_VERSION, credentialVersion); + } if (StringUtils.isNotBlank(tokenAudience)) { customClaimMap.put(AuthTokenScopes.CLAIM_TOKEN_AUDIENCE, tokenAudience); } @@ -451,12 +465,24 @@ public class AccountServiceImpl implements AccountService { return JsonWebTokenUtil.issueJwt(userId, effectiveExpire, roles, customClaimMap); } - private String issueRefreshToken(String userId, Long expirationMillis) { - Map customClaimMap = new HashMap<>(1); + private String issueRefreshToken(String userId, Long expirationMillis, Long credentialVersion) { + Map customClaimMap = new HashMap<>(2); customClaimMap.put(REFRESH_CLAIM, true); + if (credentialVersion != null) { + customClaimMap.put(ObservabilityAccessTokenGateway.CLAIM_CREDENTIAL_VERSION, credentialVersion); + } return JsonWebTokenUtil.issueJwt(userId, expirationMillis, customClaimMap); } + private static Long credentialVersion(SurenessAccount account) { + return account instanceof VersionedAccount versioned ? versioned.credentialVersion() : null; + } + + private static boolean credentialVersionMatches(SurenessAccount account, Long claimedVersion) { + return !(account instanceof VersionedAccount versioned) + || claimedVersion != null && claimedVersion == versioned.credentialVersion(); + } + private static String maskToken(String token) { return token.substring(0, 4) + "****" + token.substring(token.length() - 4); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/AccountCredentialVerifier.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/AccountCredentialVerifier.java new file mode 100644 index 0000000000..81516d2a0f --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/AccountCredentialVerifier.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import com.usthe.sureness.provider.SurenessAccount; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.HexFormat; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Component; + +/** Single password-verification policy shared by form and BASIC authentication. */ +@Component +public final class AccountCredentialVerifier { + private final IdentityPasswordPolicy passwords; + + public AccountCredentialVerifier(IdentityPasswordPolicy passwords) { + this.passwords = passwords; + } + + public boolean matches(SurenessAccount account, String supplied) { + if (supplied == null) { + return false; + } + char[] copy = supplied.toCharArray(); + try { + return matches(account, copy); + } finally { + Arrays.fill(copy, '\0'); + } + } + + public boolean matches(SurenessAccount account, char[] supplied) { + if (account == null || StringUtils.isBlank(account.getPassword()) || supplied == null) { + return false; + } + if (account instanceof VersionedAccount versioned && versioned.bcryptPassword()) { + return passwords.matches(CharBuffer.wrap(supplied), account.getPassword()); + } + if (StringUtils.isBlank(account.getSalt())) { + return account.getPassword().contentEquals(CharBuffer.wrap(supplied)); + } + return account.getPassword().equals(legacyMd5(supplied, account.getSalt())); + } + + public boolean usable(SurenessAccount account) { + return account != null && !account.isDisabledAccount() && !account.isExcessiveAttempts(); + } + + private static String legacyMd5(char[] supplied, String salt) { + ByteBuffer encodedPassword = StandardCharsets.UTF_8.encode(CharBuffer.wrap(supplied)); + byte[] encodedSalt = salt.getBytes(StandardCharsets.UTF_8); + try { + MessageDigest md5 = MessageDigest.getInstance("MD5"); + md5.update(encodedPassword); + md5.update(encodedSalt); + return HexFormat.of().withUpperCase().formatHex(md5.digest()); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("MD5 is unavailable for legacy credential migration", exception); + } finally { + if (encodedPassword.hasArray()) { + Arrays.fill(encodedPassword.array(), (byte) 0); + } + Arrays.fill(encodedSalt, (byte) 0); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/AdministratorCredentials.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/AdministratorCredentials.java new file mode 100644 index 0000000000..c84acf5946 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/AdministratorCredentials.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import java.util.Arrays; +import org.apache.commons.lang3.StringUtils; + +/** Write-only administrator credential input. */ +public final class AdministratorCredentials implements AutoCloseable { + private final String username; + private final char[] password; + + public AdministratorCredentials(String username, char[] password) { + String normalizedUsername = StringUtils.trimToNull(username); + if (normalizedUsername == null) { + throw new IllegalArgumentException("Administrator username is required"); + } + if (password == null || password.length == 0) { + throw new IllegalArgumentException("Administrator password is required"); + } + this.username = normalizedUsername; + this.password = password.clone(); + } + + String username() { + return username; + } + + char[] copyPassword() { + return password.clone(); + } + + @Override + public void close() { + Arrays.fill(password, '\0'); + } + + @Override + public String toString() { + return "AdministratorCredentials[username=" + username + ", password=redacted]"; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/BcryptPasswordProcessor.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/BcryptPasswordProcessor.java new file mode 100644 index 0000000000..48cd8a9274 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/BcryptPasswordProcessor.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import com.usthe.sureness.processor.exception.DisabledAccountException; +import com.usthe.sureness.processor.exception.ExcessiveAttemptsException; +import com.usthe.sureness.processor.exception.IncorrectCredentialsException; +import com.usthe.sureness.processor.exception.UnknownAccountException; +import com.usthe.sureness.processor.support.PasswordProcessor; +import com.usthe.sureness.provider.SurenessAccount; +import com.usthe.sureness.provider.SurenessAccountProvider; +import com.usthe.sureness.subject.Subject; + +/** Adds BCrypt verification to Sureness BASIC authentication while retaining legacy verification. */ +final class BcryptPasswordProcessor extends PasswordProcessor { + private final SurenessAccountProvider accounts; + private final AccountCredentialVerifier verifier; + + BcryptPasswordProcessor(SurenessAccountProvider accounts, AccountCredentialVerifier verifier) { + this.accounts = accounts; + this.verifier = verifier; + setAccountProvider(accounts); + } + + @Override + public Subject authenticated(Subject subject) { + SurenessAccount account = accounts.loadAccount(String.valueOf(subject.getPrincipal())); + if (account == null) { + throw new UnknownAccountException("account does not exist"); + } + if (!verifier.matches(account, subject.getCredential() == null ? null : String.valueOf(subject.getCredential()))) { + throw new IncorrectCredentialsException("incorrect password"); + } + if (account.isDisabledAccount()) { + throw new DisabledAccountException("account is disabled"); + } + if (account.isExcessiveAttempts()) { + throw new ExcessiveAttemptsException("account attempts exceeded"); + } + subject.setOwnRoles(account.getOwnRoles()); + return subject; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/BootstrapIdentityConflict.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/BootstrapIdentityConflict.java new file mode 100644 index 0000000000..4299dfb359 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/BootstrapIdentityConflict.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import java.sql.SQLException; +import org.springframework.dao.DataIntegrityViolationException; + +/** Stable application failure for a concurrent or repeated bootstrap identity transition. */ +public final class BootstrapIdentityConflict extends IllegalStateException { + public BootstrapIdentityConflict() { + super("Administrator identity is already initialized"); + } + + static RuntimeException map(DataIntegrityViolationException exception) { + Throwable current = exception; + while (current != null) { + if (current instanceof SQLException sqlException && isUniqueViolation(sqlException)) { + return new BootstrapIdentityConflict(); + } + current = current.getCause(); + } + return exception; + } + + private static boolean isUniqueViolation(SQLException exception) { + return "23505".equals(exception.getSQLState()) + || ("23000".equals(exception.getSQLState()) && exception.getErrorCode() == 1062); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/CredentialRevocation.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/CredentialRevocation.java new file mode 100644 index 0000000000..da929c76eb --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/CredentialRevocation.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +/** Revokes persisted bearer credentials after an identity credential transition. */ +public interface CredentialRevocation { + void revokeFor(String username); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/DatabaseAccount.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/DatabaseAccount.java new file mode 100644 index 0000000000..fbe9c2f5e1 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/DatabaseAccount.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import jakarta.persistence.UniqueConstraint; +import java.util.Arrays; +import java.util.List; +import org.apache.commons.lang3.StringUtils; + +/** Persisted authentication identity. The password hash is deliberately never exposed by a DTO. */ +@Entity +@Table(name = "hzb_account", uniqueConstraints = { + @UniqueConstraint(name = "uk_hzb_account_username", columnNames = "username"), + @UniqueConstraint(name = "uk_hzb_account_bootstrap", columnNames = "bootstrap_slot")}) +public class DatabaseAccount { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(nullable = false, length = 64) + private String username; + + @Column(name = "password_hash", nullable = false, length = 100) + private String passwordHash; + + @Column(nullable = false, length = 128) + private String roles; + + @Column(name = "credential_version", nullable = false) + private long credentialVersion; + + @Column(nullable = false) + private boolean disabled; + + @Column(name = "bootstrap_slot") + private Short bootstrapSlot; + + protected DatabaseAccount() { + } + + DatabaseAccount(String username, String passwordHash, String roles, long credentialVersion, + Short bootstrapSlot) { + this.username = username; + this.passwordHash = passwordHash; + this.roles = roles; + this.credentialVersion = credentialVersion; + this.bootstrapSlot = bootstrapSlot; + } + + static DatabaseAccount firstAdministrator(String username, String passwordHash, String roles) { + return new DatabaseAccount(username, passwordHash, roles, 1, (short) 1); + } + + static DatabaseAccount ordinary(String username, String passwordHash, String roles) { + return new DatabaseAccount(username, passwordHash, roles, 1, null); + } + + public String username() { + return username; + } + + String passwordHash() { + return passwordHash; + } + + String roles() { + return roles; + } + + List roleList() { + if (StringUtils.isBlank(roles)) { + return List.of(); + } + return Arrays.stream(roles.split(",")) + .map(String::trim) + .filter(StringUtils::isNotEmpty) + .toList(); + } + + public long credentialVersion() { + return credentialVersion; + } + + boolean disabled() { + return disabled; + } + + boolean bootstrapAdministrator() { + return bootstrapSlot != null; + } + + void replacePassword(String hash) { + passwordHash = hash; + credentialVersion++; + } + + @Override + public String toString() { + return "DatabaseAccount[username=" + username + ", credentialVersion=" + credentialVersion + + ", disabled=" + disabled + "]"; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/DatabaseAccountRepository.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/DatabaseAccountRepository.java new file mode 100644 index 0000000000..a103a9ed65 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/DatabaseAccountRepository.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import jakarta.persistence.LockModeType; +import java.util.Optional; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Lock; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; + +/** Persistence port for authentication identities. */ +public interface DatabaseAccountRepository extends JpaRepository { + Optional findByUsername(String username); + + /** Serializes password changes so every committed change advances the credential generation. */ + @Lock(LockModeType.PESSIMISTIC_WRITE) + @Query("SELECT account FROM DatabaseAccount account WHERE account.username = :username") + Optional findByUsernameForUpdate(@Param("username") String username); + + boolean existsByUsername(String username); + + boolean existsByBootstrapSlotIsNotNull(); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/DatabaseFirstAccountProvider.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/DatabaseFirstAccountProvider.java new file mode 100644 index 0000000000..246acf92a1 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/DatabaseFirstAccountProvider.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import com.usthe.sureness.provider.SurenessAccount; +import com.usthe.sureness.provider.SurenessAccountProvider; +import java.util.List; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Component; + +/** + * Resolves a persisted identity before the legacy source for the same username. A migrated + * identity must be disabled instead of physically deleted so its legacy definition cannot + * reappear; unmigrated custom legacy identities remain usable during an upgrade. + */ +@Component +@Primary +public class DatabaseFirstAccountProvider implements SurenessAccountProvider { + + private final DatabaseAccountRepository accounts; + private final LegacyAccountSource legacyAccounts; + + public DatabaseFirstAccountProvider(DatabaseAccountRepository accounts, LegacyAccountSource legacyAccounts) { + this.accounts = accounts; + this.legacyAccounts = legacyAccounts; + } + + @Override + public SurenessAccount loadAccount(String username) { + SurenessAccount persisted = accounts.findByUsername(username).map(PersistedAccount::new) + .orElse(null); + if (persisted != null) { + return persisted; + } + SurenessAccount legacy = legacyAccounts.loadAccount(username); + return isLegacyDefault(legacy) ? new MigrationRequiredAccount(legacy) : legacy; + } + + /** HTTP Digest cannot derive its response from a persisted BCrypt credential. */ + SurenessAccount loadLegacyAccountForDigest(String username) { + SurenessAccount account = loadAccount(username); + return account instanceof VersionedAccount ? null : account; + } + + private static boolean isLegacyDefault(SurenessAccount account) { + return account != null && "admin".equals(account.getAppId()) && "hertzbeat".equals(account.getPassword()) + && (account.getSalt() == null || account.getSalt().isBlank()); + } + + private record MigrationRequiredAccount(SurenessAccount legacy) implements SurenessAccount { + @Override + public String getAppId() { + return legacy.getAppId(); + } + + @Override + public String getPassword() { + return legacy.getPassword(); + } + + @Override + public String getSalt() { + return legacy.getSalt(); + } + + @Override + public List getOwnRoles() { + return legacy.getOwnRoles(); + } + + @Override + public boolean isDisabledAccount() { + return true; + } + + @Override + public boolean isExcessiveAttempts() { + return false; + } + + @Override + public String toString() { + return "MigrationRequiredAccount[username=admin]"; + } + } + + private record PersistedAccount(DatabaseAccount account) implements VersionedAccount { + @Override + public String getAppId() { + return account.username(); + } + + @Override + public String getPassword() { + return account.passwordHash(); + } + + @Override + public String getSalt() { + return null; + } + + @Override + public List getOwnRoles() { + return account.roleList(); + } + + @Override + public boolean isDisabledAccount() { + return account.disabled(); + } + + @Override + public boolean isExcessiveAttempts() { + return false; + } + + @Override + public long credentialVersion() { + return account.credentialVersion(); + } + + @Override + public boolean bcryptPassword() { + return true; + } + + @Override + public String toString() { + return "PersistedAccount[username=" + account.username() + "]"; + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/DatabaseIdentityProcessorConfiguration.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/DatabaseIdentityProcessorConfiguration.java new file mode 100644 index 0000000000..ef08de7c19 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/DatabaseIdentityProcessorConfiguration.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import com.usthe.sureness.configuration.SurenessProperties; +import com.usthe.sureness.configuration.SurenessProperties.AuthType; +import com.usthe.sureness.processor.DefaultProcessorManager; +import com.usthe.sureness.processor.Processor; +import com.usthe.sureness.processor.ProcessorManager; +import com.usthe.sureness.processor.support.DigestProcessor; +import com.usthe.sureness.processor.support.NoneProcessor; +import com.usthe.sureness.processor.support.SessionProcessor; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** Owns Sureness processor construction so BASIC authentication understands persisted BCrypt credentials. */ +@Configuration(proxyBeanMethods = false) +public class DatabaseIdentityProcessorConfiguration { + @Bean + ProcessorManager databaseIdentityProcessorManager(SurenessProperties properties, + DatabaseFirstAccountProvider accounts, + AccountCredentialVerifier verifier) { + List processors = new ArrayList<>(); + processors.add(new NoneProcessor()); + Set authTypes = authTypes(properties); + if (authTypes.contains(AuthType.JWT)) { + processors.add(new VersionedJwtProcessor(accounts)); + } + if (authTypes.contains(AuthType.BASIC)) { + processors.add(new BcryptPasswordProcessor(accounts, verifier)); + } + if (authTypes.contains(AuthType.DIGEST)) { + DigestProcessor digest = new DigestProcessor(); + digest.setAccountProvider(accounts::loadLegacyAccountForDigest); + processors.add(digest); + } + if (properties.getSession() != null && properties.getSession().isEnable()) { + processors.add(new SessionProcessor()); + } + return new DefaultProcessorManager(processors); + } + + static Set authTypes(SurenessProperties properties) { + if (properties.getAuths() == null || properties.getAuths().length == 0) { + return Set.of(AuthType.BASIC, AuthType.JWT); + } + return new HashSet<>(Arrays.asList(properties.getAuths())); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/IdentityInitializationService.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/IdentityInitializationService.java new file mode 100644 index 0000000000..27b8001852 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/IdentityInitializationService.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import java.util.Arrays; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +/** Owns the atomic first-administrator and explicit credential migration transitions. */ +@Service +public class IdentityInitializationService { + private final DatabaseAccountRepository accounts; + private final CredentialRevocation revocation; + private final IdentityPasswordPolicy passwords; + + public IdentityInitializationService(DatabaseAccountRepository accounts, CredentialRevocation revocation, + IdentityPasswordPolicy passwords) { + this.accounts = accounts; + this.revocation = revocation; + this.passwords = passwords; + } + + @Transactional + public void createFirstAdministrator(AdministratorCredentials credentials) { + char[] clear = credentials.copyPassword(); + try { + if (accounts.existsByBootstrapSlotIsNotNull() + || accounts.existsByUsername(credentials.username())) { + throw new BootstrapIdentityConflict(); + } + try { + accounts.saveAndFlush(DatabaseAccount.firstAdministrator( + credentials.username(), passwords.encode(clear), "admin")); + } catch (DataIntegrityViolationException exception) { + throw BootstrapIdentityConflict.map(exception); + } + } finally { + Arrays.fill(clear, '\0'); + credentials.close(); + } + } + + /** Changes a password atomically and clears the caller-supplied character array. */ + @Transactional + public void changePassword(String username, char[] password) { + char[] clear = password == null ? new char[0] : password.clone(); + try { + if (clear.length == 0) { + throw new IllegalArgumentException("Password is required"); + } + DatabaseAccount account = accounts.findByUsernameForUpdate(username) + .orElseThrow(() -> new IllegalArgumentException("Account does not exist")); + account.replacePassword(passwords.encode(clear)); + accounts.save(account); + revocation.revokeFor(username); + } finally { + Arrays.fill(clear, '\0'); + if (password != null) { + Arrays.fill(password, '\0'); + } + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/IdentityPasswordPolicy.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/IdentityPasswordPolicy.java new file mode 100644 index 0000000000..ae83193698 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/IdentityPasswordPolicy.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import java.nio.CharBuffer; +import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; +import org.springframework.stereotype.Component; + +/** Owns the persisted identity password algorithm and work factor. */ +@Component +public final class IdentityPasswordPolicy { + static final int BCRYPT_COST = 12; + private final BCryptPasswordEncoder bcrypt = new BCryptPasswordEncoder(BCRYPT_COST); + + public String encode(char[] clearPassword) { + return bcrypt.encode(CharBuffer.wrap(clearPassword)); + } + + public boolean matches(CharSequence supplied, String hash) { + return bcrypt.matches(supplied, hash); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/LegacyAccountMigrationService.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/LegacyAccountMigrationService.java new file mode 100644 index 0000000000..a57c2820ff --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/LegacyAccountMigrationService.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import com.usthe.sureness.provider.SurenessAccount; +import java.util.Arrays; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +/** Explicitly replaces one verified legacy identity with a database-owned identity. */ +@Service +public class LegacyAccountMigrationService { + private final DatabaseAccountRepository accounts; + private final LegacyAccountSource legacyAccounts; + private final AccountCredentialVerifier verifier; + private final CredentialRevocation revocation; + private final IdentityPasswordPolicy passwords; + + public LegacyAccountMigrationService(DatabaseAccountRepository accounts, LegacyAccountSource legacyAccounts, + AccountCredentialVerifier verifier, CredentialRevocation revocation, + IdentityPasswordPolicy passwords) { + this.accounts = accounts; + this.legacyAccounts = legacyAccounts; + this.verifier = verifier; + this.revocation = revocation; + this.passwords = passwords; + } + + /** Replaces a legacy credential and clears both caller-supplied password arrays. */ + @Transactional + public void migrate(String username, char[] legacyPassword, char[] replacementPassword) { + char[] legacy = legacyPassword == null ? new char[0] : legacyPassword.clone(); + char[] replacement = replacementPassword == null ? new char[0] : replacementPassword.clone(); + try { + if (accounts.existsByUsername(username)) { + throw new BootstrapIdentityConflict(); + } + SurenessAccount source = legacyAccounts.loadAccount(username); + if (!verifier.matches(source, legacy)) { + throw new IllegalArgumentException("Legacy credential is invalid"); + } + if (replacement.length == 0) { + throw new IllegalArgumentException("Replacement password is required"); + } + String roles = String.join(",", source.getOwnRoles()); + try { + String passwordHash = passwords.encode(replacement); + boolean claimsBootstrapAdministrator = source.getOwnRoles().contains("admin") + && !accounts.existsByBootstrapSlotIsNotNull(); + DatabaseAccount migrated = claimsBootstrapAdministrator + ? DatabaseAccount.firstAdministrator(username, passwordHash, roles) + : DatabaseAccount.ordinary(username, passwordHash, roles); + accounts.saveAndFlush(migrated); + } catch (DataIntegrityViolationException exception) { + throw BootstrapIdentityConflict.map(exception); + } + revocation.revokeFor(username); + } finally { + Arrays.fill(legacy, '\0'); + Arrays.fill(replacement, '\0'); + if (legacyPassword != null) { + Arrays.fill(legacyPassword, '\0'); + } + if (replacementPassword != null) { + Arrays.fill(replacementPassword, '\0'); + } + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/LegacyAccountSource.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/LegacyAccountSource.java new file mode 100644 index 0000000000..46d75ec752 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/LegacyAccountSource.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import com.usthe.sureness.provider.SurenessAccount; +import com.usthe.sureness.provider.SurenessAccountProvider; +import com.usthe.sureness.provider.ducument.DocumentAccountProvider; +import org.springframework.stereotype.Component; + +/** Isolates the upgrade-only Sureness document identity source. */ +@Component +public class LegacyAccountSource implements SurenessAccountProvider { + private final SurenessAccountProvider documents = new DocumentAccountProvider(); + + @Override + public SurenessAccount loadAccount(String username) { + return documents.loadAccount(username); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/PersistedTokenCredentialRevocation.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/PersistedTokenCredentialRevocation.java new file mode 100644 index 0000000000..87369f9a5e --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/PersistedTokenCredentialRevocation.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import java.time.LocalDateTime; +import org.apache.hertzbeat.manager.dao.AuthTokenDao; +import org.springframework.stereotype.Component; + +/** Revokes database-managed bearer tokens; UI JWTs are revoked by the credential-version claim. */ +@Component +public class PersistedTokenCredentialRevocation implements CredentialRevocation { + private final AuthTokenDao tokens; + + public PersistedTokenCredentialRevocation(AuthTokenDao tokens) { + this.tokens = tokens; + } + + @Override + public void revokeFor(String username) { + tokens.revokeActiveByCreator(username, username, LocalDateTime.now()); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/VersionedAccount.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/VersionedAccount.java new file mode 100644 index 0000000000..b19ce88768 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/VersionedAccount.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import com.usthe.sureness.provider.SurenessAccount; + +/** Account contract used to invalidate stateless sessions after credential changes. */ +public interface VersionedAccount extends SurenessAccount { + long credentialVersion(); + + boolean bcryptPassword(); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/VersionedJwtProcessor.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/VersionedJwtProcessor.java new file mode 100644 index 0000000000..6ea33d7652 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/VersionedJwtProcessor.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import com.usthe.sureness.processor.exception.ExpiredCredentialsException; +import com.usthe.sureness.processor.support.JwtProcessor; +import com.usthe.sureness.provider.SurenessAccount; +import com.usthe.sureness.provider.SurenessAccountProvider; +import com.usthe.sureness.subject.Subject; +import com.usthe.sureness.util.JsonWebTokenUtil; +import io.jsonwebtoken.Claims; +import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes; +import org.apache.hertzbeat.common.observability.gateway.ObservabilityAccessTokenGateway; + +/** Enforces credential generations when UI-session JWTs enter through Sureness directly. */ +final class VersionedJwtProcessor extends JwtProcessor { + private final SurenessAccountProvider accounts; + + VersionedJwtProcessor(SurenessAccountProvider accounts) { + this.accounts = accounts; + } + + @Override + public Subject authenticated(Subject subject) { + Subject authenticated = super.authenticated(subject); + Claims claims = JsonWebTokenUtil.parseJwt(String.valueOf(subject.getCredential())); + if (!AuthTokenScopes.UI_SESSION.equals(claims.get(AuthTokenScopes.CLAIM_TOKEN_SCOPE, String.class))) { + return authenticated; + } + SurenessAccount account = accounts.loadAccount(claims.getSubject()); + requireUsable(account); + Long claimed = claims.get(ObservabilityAccessTokenGateway.CLAIM_CREDENTIAL_VERSION, Long.class); + if (account instanceof VersionedAccount versioned + && (claimed == null || claimed != versioned.credentialVersion())) { + throw new ExpiredCredentialsException("session credentials are outdated"); + } + return authenticated; + } + + private static void requireUsable(SurenessAccount account) { + if (account == null || account.isDisabledAccount() || account.isExcessiveAttempts()) { + throw new ExpiredCredentialsException("session account is unavailable"); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/DatabasePresence.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/DatabasePresence.java new file mode 100644 index 0000000000..cb50e1a58d --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/DatabasePresence.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.installation; + +/** Result of a bounded database reachability/schema probe. */ +public enum DatabasePresence { EMPTY, HERTZBEAT_SCHEMA, UNREACHABLE } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationClassifier.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationClassifier.java new file mode 100644 index 0000000000..69d0c735f7 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationClassifier.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.installation; + +import java.util.Optional; + +/** Pure startup decision boundary; unreachable configured persistence can never become setup. */ +public final class InstallationClassifier { + public InstallationMode classify(DatabasePresence database, Optional record, + Optional localFingerprint) { + if (database == DatabasePresence.UNREACHABLE) { + return InstallationMode.RECOVERY; + } + if (database == DatabasePresence.EMPTY) { + return localFingerprint.isEmpty() ? InstallationMode.SETUP : InstallationMode.RECOVERY; + } + if (record.isEmpty()) { + return InstallationMode.UPGRADE; + } + InstallationRecord installed = record.orElseThrow(); + if (!installed.complete() || localFingerprint.isEmpty() + || !installed.fingerprint().equals(localFingerprint.orElseThrow().value())) { + return InstallationMode.RECOVERY; + } + return InstallationMode.FULL; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationCompletionService.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationCompletionService.java new file mode 100644 index 0000000000..9ea9b43db7 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationCompletionService.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.installation; + +import java.util.Optional; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.stereotype.Service; + +/** Commits the permanent close of setup writes, with idempotence only for the same installation. */ +@Service +public class InstallationCompletionService { + private final InstallationRecordRepository records; + + public InstallationCompletionService(InstallationRecordRepository records) { + this.records = records; + } + + public void complete(InstallationFingerprint fingerprint) { + Optional existing = records.findById(InstallationRecord.SINGLETON_ID); + if (existing.isPresent()) { + requireSameInstallation(existing.orElseThrow(), fingerprint); + return; + } + try { + records.saveAndFlush(new InstallationRecord(fingerprint.value())); + } catch (DataIntegrityViolationException conflict) { + Optional concurrent = records.findById(InstallationRecord.SINGLETON_ID); + if (concurrent.isEmpty()) { + throw conflict; + } + requireSameInstallation(concurrent.orElseThrow(), fingerprint); + } + } + + public boolean writesClosed() { + return records.existsById(InstallationRecord.SINGLETON_ID); + } + + private static void requireSameInstallation(InstallationRecord existing, InstallationFingerprint expected) { + if (!existing.complete() || !existing.fingerprint().equals(expected.value())) { + throw new IllegalStateException("Installation identity does not match"); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationFingerprint.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationFingerprint.java new file mode 100644 index 0000000000..634aa2e6dc --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationFingerprint.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.installation; + +import java.util.Objects; + +/** Opaque local identity paired with the database installation record. */ +public record InstallationFingerprint(String value) { + public InstallationFingerprint { + Objects.requireNonNull(value, "value"); + if (!value.matches("[a-f0-9]{64}")) { + throw new IllegalArgumentException("Installation fingerprint is invalid"); + } + } + + @Override + public String toString() { + return "InstallationFingerprint[redacted]"; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationMode.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationMode.java new file mode 100644 index 0000000000..3c2450ade5 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationMode.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.installation; + +/** Fail-closed startup classification. */ +public enum InstallationMode { SETUP, UPGRADE, FULL, RECOVERY } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationRecord.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationRecord.java new file mode 100644 index 0000000000..280e6d39df --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationRecord.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.installation; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; + +/** Permanent marker proving that setup completed for one local installation. */ +@Entity +@Table(name = "hzb_installation") +public class InstallationRecord { + static final short SINGLETON_ID = 1; + + @Id + private Short id; + @Column(name = "installation_fingerprint", nullable = false, length = 64, unique = true) + private String fingerprint; + @Column(nullable = false) + private boolean complete; + + protected InstallationRecord() { + } + + InstallationRecord(String fingerprint) { + this.id = SINGLETON_ID; + this.fingerprint = fingerprint; + this.complete = true; + } + + String fingerprint() { + return fingerprint; + } + + boolean complete() { + return complete; + } + + @Override + public String toString() { + return "InstallationRecord[complete=" + complete + "]"; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationRecordRepository.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationRecordRepository.java new file mode 100644 index 0000000000..60c9dbbaaa --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationRecordRepository.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.installation; + +import org.springframework.data.jpa.repository.JpaRepository; + +/** Persistence port for the singleton installation record. */ +public interface InstallationRecordRepository extends JpaRepository { +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/LocalInstallationFingerprintStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/LocalInstallationFingerprintStore.java new file mode 100644 index 0000000000..6d03d53281 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/LocalInstallationFingerprintStore.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.installation; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.SecureRandom; +import java.util.Arrays; +import java.util.HexFormat; +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; + +/** Owner-only local installation identity store. */ +public final class LocalInstallationFingerprintStore { + private final Path path; + private final SecureRandom random; + + public LocalInstallationFingerprintStore(Path path, SecureRandom random) { + this.path = path.toAbsolutePath().normalize(); + this.random = random; + } + + public Optional read() throws IOException { + if (!SecureSetupFile.isOwnerOnlyRegularFile(path)) { + return Optional.empty(); + } + return Optional.of(new InstallationFingerprint(Files.readString(path, StandardCharsets.US_ASCII).trim())); + } + + public InstallationFingerprint create() throws IOException { + byte[] value = new byte[32]; + random.nextBytes(value); + InstallationFingerprint fingerprint = new InstallationFingerprint(HexFormat.of().formatHex(value)); + try { + SecureSetupFile.ensureSafeParent(path); + SecureSetupFile.create(path, fingerprint.value().getBytes(StandardCharsets.US_ASCII)); + return fingerprint; + } finally { + Arrays.fill(value, (byte) 0); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/RemoteSetupUnlock.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/RemoteSetupUnlock.java new file mode 100644 index 0000000000..9e0c51ca1a --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/RemoteSetupUnlock.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.security; + +import java.io.IOException; +import java.net.InetAddress; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.Arrays; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** One-time local-file proof required when setup listens beyond loopback. */ +public final class RemoteSetupUnlock { + private static final Logger LOGGER = LoggerFactory.getLogger(RemoteSetupUnlock.class); + private static final Duration TTL = Duration.ofMinutes(15); + private static final int MAX_ATTEMPTS = 5; + private static final int MAX_CLIENTS = 1024; + private final Path codeFile; + private final Clock clock; + private final SecureRandom random; + private final Map attempts = new HashMap<>(); + private byte[] codeDigest; + private byte[] sessionDigest; + private Instant expiresAt; + + public RemoteSetupUnlock(Path codeFile, Clock clock, SecureRandom random) { + this.codeFile = codeFile.toAbsolutePath().normalize(); + this.clock = clock; + this.random = random; + } + + public boolean requiresUnlock(InetAddress bindAddress) { + return !bindAddress.isLoopbackAddress(); + } + + public synchronized void open() throws IOException { + SecureSetupFile.ensureSafeParent(codeFile); + removeStaleCodeFile(); + clearDigest(sessionDigest); + sessionDigest = null; + attempts.clear(); + byte[] entropy = new byte[24]; + byte[] encodedCode = null; + try { + random.nextBytes(entropy); + encodedCode = Base64.getUrlEncoder().withoutPadding().encode(entropy); + clearDigest(codeDigest); + codeDigest = digest(encodedCode); + expiresAt = clock.instant().plus(TTL); + SecureSetupFile.create(codeFile, encodedCode); + } catch (IOException | RuntimeException exception) { + clearDigest(codeDigest); + codeDigest = null; + throw exception; + } finally { + Arrays.fill(entropy, (byte) 0); + if (encodedCode != null) { + Arrays.fill(encodedCode, (byte) 0); + } + } + LOGGER.warn("Remote setup requires the one-time unlock file at {}", codeFile); + } + + private void removeStaleCodeFile() throws IOException { + if (!Files.exists(codeFile, LinkOption.NOFOLLOW_LINKS)) { + return; + } + if (!SecureSetupFile.isOwnerOnlyRegularFile(codeFile)) { + throw new IOException("Existing setup unlock path is not an owner-only regular file"); + } + Files.delete(codeFile); + } + + public synchronized SetupAccessSession redeem(String remoteAddress, SetupUnlockCode supplied) throws IOException { + Instant now = clock.instant(); + attempts.entrySet().removeIf(entry -> entry.getValue().expired(now)); + if (!attempts.containsKey(remoteAddress) && attempts.size() >= MAX_CLIENTS) { + throw new SetupUnlockRejected(SetupUnlockRejected.Reason.RATE_LIMITED); + } + Attempts current = attempts.compute(remoteAddress, (key, value) -> value == null || value.expired(now) + ? new Attempts(now.plus(TTL), 1) : value.increment()); + if (current.count() > MAX_ATTEMPTS) { + throw new SetupUnlockRejected(SetupUnlockRejected.Reason.RATE_LIMITED); + } + char[] value = supplied.copyValue(); + byte[] suppliedDigest = null; + try { + suppliedDigest = digest(value); + if (codeDigest == null) { + throw new SetupUnlockRejected(SetupUnlockRejected.Reason.INVALID); + } + if (!now.isBefore(expiresAt)) { + throw new SetupUnlockRejected(SetupUnlockRejected.Reason.EXPIRED); + } + if (!MessageDigest.isEqual(codeDigest, suppliedDigest)) { + throw new SetupUnlockRejected(SetupUnlockRejected.Reason.INVALID); + } + byte[] entropy = new byte[32]; + byte[] newSessionDigest = null; + try { + random.nextBytes(entropy); + String token = Base64.getUrlEncoder().withoutPadding().encodeToString(entropy); + newSessionDigest = digest(token); + + // Publish the in-memory session only after the one-time proof is durably unavailable. + Files.deleteIfExists(codeFile); + clearDigest(sessionDigest); + sessionDigest = newSessionDigest; + newSessionDigest = null; + clearDigest(codeDigest); + codeDigest = null; + return new SetupAccessSession(token, expiresAt); + } finally { + Arrays.fill(entropy, (byte) 0); + clearDigest(newSessionDigest); + } + } finally { + clearDigest(suppliedDigest); + Arrays.fill(value, '\0'); + supplied.close(); + } + } + + public synchronized boolean permits(String token) { + if (token == null || sessionDigest == null || !clock.instant().isBefore(expiresAt)) { + return false; + } + byte[] suppliedDigest = digest(token); + try { + return MessageDigest.isEqual(sessionDigest, suppliedDigest); + } finally { + Arrays.fill(suppliedDigest, (byte) 0); + } + } + + public synchronized void close() throws IOException { + clearDigest(codeDigest); + clearDigest(sessionDigest); + codeDigest = null; + sessionDigest = null; + attempts.clear(); + Files.deleteIfExists(codeFile); + } + + private static byte[] digest(String value) { + byte[] encoded = value.getBytes(StandardCharsets.UTF_8); + try { + return digest(encoded); + } finally { + Arrays.fill(encoded, (byte) 0); + } + } + + private static byte[] digest(byte[] value) { + try { + return MessageDigest.getInstance("SHA-256").digest(value); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("SHA-256 is unavailable", exception); + } + } + + private static byte[] digest(char[] value) { + ByteBuffer encoded = StandardCharsets.UTF_8.encode(CharBuffer.wrap(value)); + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + digest.update(encoded); + return digest.digest(); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("SHA-256 is unavailable", exception); + } finally { + if (encoded.hasArray()) { + Arrays.fill(encoded.array(), (byte) 0); + } + } + } + + private static void clearDigest(byte[] digest) { + if (digest != null) { + Arrays.fill(digest, (byte) 0); + } + } + + private record Attempts(Instant resetsAt, int count) { + Attempts increment() { + return new Attempts(resetsAt, count + 1); + } + + boolean expired(Instant now) { + return !now.isBefore(resetsAt); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFile.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFile.java new file mode 100644 index 0000000000..555b04dd83 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFile.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.security; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Set; + +/** Creates a new local secret without following links or exposing permissive content. */ +public final class SecureSetupFile { + private static final Set OWNER_READ_WRITE = + PosixFilePermissions.fromString("rw-------"); + + private SecureSetupFile() { + } + + public static void ensureSafeParent(Path target) throws IOException { + Path parent = target.getParent(); + Files.createDirectories(parent); + if (Files.isSymbolicLink(parent)) { + throw new IOException("Setup secret parent must not be a symbolic link"); + } + } + + public static void create(Path target, byte[] content) throws IOException { + Path resolvedTarget = target.getParent().toRealPath().resolve(target.getFileName()); + if (!Files.getFileStore(resolvedTarget.getParent()).supportsFileAttributeView("posix")) { + throw new IOException("Owner-only setup secrets require POSIX file permissions"); + } + try (FileChannel channel = FileChannel.open(resolvedTarget, + Set.of(StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE, LinkOption.NOFOLLOW_LINKS), + PosixFilePermissions.asFileAttribute(OWNER_READ_WRITE))) { + ByteBuffer buffer = ByteBuffer.wrap(content); + while (buffer.hasRemaining()) { + channel.write(buffer); + } + channel.force(true); + } + } + + public static boolean isOwnerOnlyRegularFile(Path target) throws IOException { + if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) + || !Files.getFileStore(target).supportsFileAttributeView("posix")) { + return false; + } + return Files.getPosixFilePermissions(target, LinkOption.NOFOLLOW_LINKS).equals(OWNER_READ_WRITE); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupAccessCookie.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupAccessCookie.java new file mode 100644 index 0000000000..730c1afab5 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupAccessCookie.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.security; + +import java.time.Duration; +import java.time.Clock; +import org.springframework.http.ResponseCookie; + +/** Transport policy for the remote setup capability. */ +public final class SetupAccessCookie { + public static final String NAME = "hertzbeat_setup"; + + private SetupAccessCookie() { + } + + public static ResponseCookie create(SetupAccessSession session, boolean secure, Clock clock) { + long seconds = Math.max(1, Duration.between(clock.instant(), session.expiresAt()).getSeconds()); + return ResponseCookie.from(NAME, session.token()).httpOnly(true).sameSite("Strict").secure(secure) + .path("/api/setup").maxAge(Duration.ofSeconds(seconds)).build(); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupAccessSession.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupAccessSession.java new file mode 100644 index 0000000000..1c94022e59 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupAccessSession.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.security; + +import java.time.Instant; + +/** Opaque setup cookie value returned exactly once after unlock. */ +public final class SetupAccessSession { + private final String token; + private final Instant expiresAt; + + SetupAccessSession(String token, Instant expiresAt) { + this.token = token; + this.expiresAt = expiresAt; + } + + String token() { + return token; + } + + public Instant expiresAt() { + return expiresAt; + } + + @Override + public String toString() { + return "SetupAccessSession[token=redacted, expiresAt=" + expiresAt + "]"; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupUnlockCode.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupUnlockCode.java new file mode 100644 index 0000000000..754e6ab89d --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupUnlockCode.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.security; + +import java.util.Arrays; + +/** Write-only unlock proof. */ +public final class SetupUnlockCode implements AutoCloseable { + private final char[] value; + + public SetupUnlockCode(char[] value) { + this.value = value == null ? new char[0] : value.clone(); + } + + char[] copyValue() { + return value.clone(); + } + + @Override + public void close() { + Arrays.fill(value, '\0'); + } + + @Override + public String toString() { + return "SetupUnlockCode[redacted]"; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupUnlockRejected.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupUnlockRejected.java new file mode 100644 index 0000000000..a6d2400995 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupUnlockRejected.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.security; + +/** Typed setup unlock failure that an API boundary can map without parsing exception text. */ +public final class SetupUnlockRejected extends IllegalStateException { + /** Stable rejection categories exposed to the setup HTTP boundary. */ + public enum Reason { + INVALID, + EXPIRED, + RATE_LIMITED + } + + private final Reason reason; + + SetupUnlockRejected(Reason reason) { + super(safeMessage(reason)); + this.reason = reason; + } + + public Reason reason() { + return reason; + } + + private static String safeMessage(Reason reason) { + return switch (reason) { + case INVALID -> "Setup unlock proof is invalid"; + case EXPIRED -> "Setup unlock proof is expired"; + case RATE_LIMITED -> "Setup unlock attempts exceeded"; + }; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/ui/session/UiSessionService.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/ui/session/UiSessionService.java index f203f0068b..5da013ad3f 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/ui/session/UiSessionService.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/ui/session/UiSessionService.java @@ -27,6 +27,7 @@ import java.util.Map; import javax.naming.AuthenticationException; import org.apache.commons.lang3.StringUtils; import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes; +import org.apache.hertzbeat.common.observability.gateway.ObservabilityAccessTokenGateway; import org.apache.hertzbeat.manager.pojo.dto.LoginDto; import org.apache.hertzbeat.manager.pojo.dto.RefreshTokenResponse; import org.apache.hertzbeat.manager.service.AccountService; @@ -75,7 +76,9 @@ public class UiSessionService { return UiSessionView.anonymous(); } List roles = roles(claims.get(ROLES_CLAIM)); - if (accountService.checkManagedTokenAccess(username, roles) != null) { + Long credentialVersion = claims.get( + ObservabilityAccessTokenGateway.CLAIM_CREDENTIAL_VERSION, Long.class); + if (accountService.checkSessionAccess(username, roles, credentialVersion) != null) { return UiSessionView.anonymous(); } String workspaceId = AuthTokenScopes.normalizeWorkspaceId( diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/ApiTokenValidationFilterTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/ApiTokenValidationFilterTest.java index d639d06d25..fe50d3956d 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/ApiTokenValidationFilterTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/ApiTokenValidationFilterTest.java @@ -36,6 +36,7 @@ import java.util.List; import org.apache.hertzbeat.common.constants.NetworkConstants; import org.apache.hertzbeat.common.observability.gateway.AuthTokenRequestContext; import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes; +import org.apache.hertzbeat.common.observability.gateway.ObservabilityAccessTokenGateway; import org.apache.hertzbeat.manager.service.AccountService; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -70,6 +71,8 @@ class ApiTokenValidationFilterTest { AuthTokenRequestContext.clear(); lenient().when(request.getHeader(AuthTokenScopes.WORKSPACE_ID_HEADER)).thenReturn(null); lenient().when(principalMap.getPrincipal(AuthTokenScopes.CLAIM_WORKSPACE_ID)).thenReturn(null); + lenient().when(principalMap.getPrincipal( + ObservabilityAccessTokenGateway.CLAIM_CREDENTIAL_VERSION)).thenReturn(null); } @AfterEach @@ -155,16 +158,17 @@ class ApiTokenValidationFilterTest { when(request.getMethod()).thenReturn("POST"); when(request.getRequestURI()).thenReturn("/api/monitor"); when(accountService.checkTokenStatus(managedToken, AuthTokenScopes.API_ADMIN)).thenReturn(null); - when(accountService.checkManagedTokenAccess("admin", List.of("admin"))).thenReturn(null); + when(accountService.checkManagedTokenAccess("admin", List.of("admin"), 7L)).thenReturn(null); doNothing().when(accountService).touchTokenLastUsedTime(managedToken); SubjectSum subject = mockManagedSubjectWithClaims(); + when(principalMap.getPrincipal(ObservabilityAccessTokenGateway.CLAIM_CREDENTIAL_VERSION)).thenReturn(7L); try (var mockedStatic = mockStatic(SurenessContextHolder.class)) { mockedStatic.when(SurenessContextHolder::getBindSubject).thenReturn(subject); org.junit.jupiter.api.Assertions.assertTrue(filter.preHandle(request, response, new Object())); verify(accountService).checkTokenStatus(managedToken, AuthTokenScopes.API_ADMIN); - verify(accountService).checkManagedTokenAccess("admin", List.of("admin")); + verify(accountService).checkManagedTokenAccess("admin", List.of("admin"), 7L); verify(accountService).touchTokenLastUsedTime(managedToken); } } @@ -176,7 +180,7 @@ class ApiTokenValidationFilterTest { when(request.getMethod()).thenReturn("GET"); when(request.getRequestURI()).thenReturn("/api/monitor"); when(accountService.checkTokenStatus(managedToken, AuthTokenScopes.READONLY_QUERY)).thenReturn(null); - when(accountService.checkManagedTokenAccess("admin", List.of("admin"))).thenReturn(null); + when(accountService.checkManagedTokenAccess("admin", List.of("admin"), null)).thenReturn(null); doNothing().when(accountService).touchTokenLastUsedTime(managedToken); SubjectSum subject = mockManagedSubjectWithClaims(); @@ -194,7 +198,7 @@ class ApiTokenValidationFilterTest { when(request.getHeader(NetworkConstants.AUTHORIZATION)).thenReturn("Bearer " + managedToken); when(request.getRequestURI()).thenReturn("/api/otlp/v1/metrics"); when(accountService.checkTokenStatus(managedToken, AuthTokenScopes.OTLP_INGEST)).thenReturn(null); - when(accountService.checkManagedTokenAccess("admin", List.of("admin"))).thenReturn(null); + when(accountService.checkManagedTokenAccess("admin", List.of("admin"), null)).thenReturn(null); doNothing().when(accountService).touchTokenLastUsedTime(managedToken); SubjectSum subject = mockManagedSubjectWithClaims(); @@ -213,7 +217,7 @@ class ApiTokenValidationFilterTest { when(request.getHeader(NetworkConstants.AUTHORIZATION)).thenReturn("Bearer " + managedToken); when(request.getRequestURI()).thenReturn("/api/otlp/v1/metrics"); when(accountService.checkTokenStatus(managedToken, AuthTokenScopes.OTLP_INGEST)).thenReturn(null); - when(accountService.checkManagedTokenAccess("admin", List.of("admin"))).thenReturn(null); + when(accountService.checkManagedTokenAccess("admin", List.of("admin"), null)).thenReturn(null); SubjectSum subject = mockManagedSubjectWithClaims(); when(principalMap.getPrincipal(AuthTokenScopes.CLAIM_TOKEN_AUDIENCE)) .thenReturn(AuthTokenScopes.MANAGED_COLLECTOR_AUDIENCE); @@ -238,7 +242,7 @@ class ApiTokenValidationFilterTest { when(request.getHeader(NetworkConstants.AUTHORIZATION)).thenReturn("Bearer " + managedToken); when(request.getRequestURI()).thenReturn("/api/otlp/v1/traces"); when(accountService.checkTokenStatus(managedToken, AuthTokenScopes.OTLP_INGEST)).thenReturn(null); - when(accountService.checkManagedTokenAccess("admin", List.of("admin"))).thenReturn(null); + when(accountService.checkManagedTokenAccess("admin", List.of("admin"), null)).thenReturn(null); when(response.getWriter()).thenReturn(new PrintWriter(new StringWriter())); SubjectSum subject = mockManagedSubjectWithClaims(); when(principalMap.getPrincipal(AuthTokenScopes.CLAIM_TOKEN_AUDIENCE)) @@ -264,7 +268,7 @@ class ApiTokenValidationFilterTest { when(request.getMethod()).thenReturn("POST"); when(request.getRequestURI()).thenReturn("/api/monitor"); when(accountService.checkTokenStatus(managedToken, AuthTokenScopes.API_ADMIN, "prod-west")).thenReturn(null); - when(accountService.checkManagedTokenAccess("admin", List.of("admin"))).thenReturn(null); + when(accountService.checkManagedTokenAccess("admin", List.of("admin"), null)).thenReturn(null); doNothing().when(accountService).touchTokenLastUsedTime(managedToken); SubjectSum subject = mockManagedSubjectWithClaims(); @@ -331,7 +335,7 @@ class ApiTokenValidationFilterTest { when(request.getMethod()).thenReturn("POST"); when(request.getRequestURI()).thenReturn("/api/monitor"); when(accountService.checkTokenStatus(managedToken, AuthTokenScopes.API_ADMIN)).thenReturn(null); - when(accountService.checkManagedTokenAccess("admin", List.of("admin"))) + when(accountService.checkManagedTokenAccess("admin", List.of("admin"), null)) .thenReturn("Token permissions are outdated"); StringWriter stringWriter = new StringWriter(); @@ -367,7 +371,7 @@ class ApiTokenValidationFilterTest { when(request.getMethod()).thenReturn("POST"); when(request.getRequestURI()).thenReturn("/api/monitor"); when(accountService.checkTokenStatus(managedToken, AuthTokenScopes.API_ADMIN)).thenReturn(null); - when(accountService.checkManagedTokenAccess("admin", List.of("admin"))).thenReturn(null); + when(accountService.checkManagedTokenAccess("admin", List.of("admin"), null)).thenReturn(null); // touchTokenLastUsedTime throws exception org.mockito.Mockito.doThrow(new RuntimeException("DB error")) .when(accountService).touchTokenLastUsedTime(managedToken); @@ -397,7 +401,7 @@ class ApiTokenValidationFilterTest { when(request.getMethod()).thenReturn("POST"); when(request.getRequestURI()).thenReturn("/api/monitor"); when(accountService.checkTokenStatus(managedToken, AuthTokenScopes.API_ADMIN)).thenReturn(null); - when(accountService.checkManagedTokenAccess("admin", List.of("admin"))) + when(accountService.checkManagedTokenAccess("admin", List.of("admin"), null)) .thenThrow(new RuntimeException("account store unavailable")); StringWriter stringWriter = new StringWriter(); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/AccountCredentialVersionTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/AccountCredentialVersionTest.java new file mode 100644 index 0000000000..f1445bbd13 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/AccountCredentialVersionTest.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.service; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +import com.usthe.sureness.provider.SurenessAccountProvider; +import com.usthe.sureness.subject.SubjectSum; +import com.usthe.sureness.util.JsonWebTokenUtil; +import com.usthe.sureness.util.SurenessContextHolder; +import io.jsonwebtoken.Claims; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import javax.naming.AuthenticationException; +import org.apache.hertzbeat.common.observability.gateway.ObservabilityAccessTokenGateway; +import org.apache.hertzbeat.manager.dao.AuthTokenDao; +import org.apache.hertzbeat.manager.pojo.dto.LoginDto; +import org.apache.hertzbeat.manager.service.impl.AccountServiceImpl; +import org.apache.hertzbeat.manager.setup.identity.AccountCredentialVerifier; +import org.apache.hertzbeat.manager.setup.identity.IdentityPasswordPolicy; +import org.apache.hertzbeat.manager.setup.identity.VersionedAccount; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; + +class AccountCredentialVersionTest { + private MutableVersionedAccount account; + private AccountServiceImpl service; + + @BeforeEach + void setUp() { + JsonWebTokenUtil.setDefaultSecretKey("long-test-key-which-is-not-a-production-secret-1234567890"); + account = new MutableVersionedAccount(); + SurenessAccountProvider provider = username -> "owner".equals(username) ? account : null; + service = new AccountServiceImpl(provider, mock(AuthTokenDao.class), + new AccountCredentialVerifier(new IdentityPasswordPolicy())); + } + + @Test + void bcryptLoginCarriesCredentialVersionOnAccessAndRefreshTokens() throws Exception { + Map issued = service.authGetToken( + LoginDto.builder().identifier("owner").credential("password").build()); + + Claims access = JsonWebTokenUtil.parseJwt(issued.get("token")); + Claims refresh = JsonWebTokenUtil.parseJwt(issued.get("refreshToken")); + assertEquals(3L, access.get("credentialVersion", Long.class)); + assertEquals(3L, refresh.get("credentialVersion", Long.class)); + assertNull(service.checkSessionAccess("owner", List.of("admin"), 3L)); + } + + @Test + void credentialChangeRejectsExistingAccessAndRefreshTokens() throws Exception { + Map issued = service.authGetToken( + LoginDto.builder().identifier("owner").credential("password").build()); + account.version = 4; + + assertEquals("Token credentials are outdated", service.checkSessionAccess("owner", List.of("admin"), 3L)); + assertThrows(AuthenticationException.class, () -> service.refreshToken(issued.get("refreshToken"))); + } + + @Test + void credentialChangeInvalidatesManagedApiTokenOwnerCheck() { + assertNull(service.checkManagedTokenAccess("owner", List.of("admin"), 3L)); + account.version = 4; + + assertEquals("Token credentials are outdated", + service.checkManagedTokenAccess("owner", List.of("admin"), 3L)); + } + + @Test + void managedTokenUsesTheAccountSnapshotThatWasAuthorized() throws Exception { + MutableVersionedAccount authorized = new MutableVersionedAccount(); + MutableVersionedAccount concurrentlyChanged = new MutableVersionedAccount(); + concurrentlyChanged.version = 4; + AtomicInteger loads = new AtomicInteger(); + SurenessAccountProvider provider = username -> loads.getAndIncrement() == 0 + ? authorized : concurrentlyChanged; + AccountServiceImpl accountService = new AccountServiceImpl(provider, mock(AuthTokenDao.class), + new AccountCredentialVerifier(new IdentityPasswordPolicy())); + SubjectSum subject = mock(SubjectSum.class); + when(subject.getPrincipal()).thenReturn("owner"); + + try (var context = mockStatic(SurenessContextHolder.class)) { + context.when(SurenessContextHolder::getBindSubject).thenReturn(subject); + String token = accountService.generateToken("automation", 3600L); + + Claims claims = JsonWebTokenUtil.parseJwt(token); + assertEquals(3L, claims.get( + ObservabilityAccessTokenGateway.CLAIM_CREDENTIAL_VERSION, Long.class)); + assertEquals(1, loads.get()); + } + } + + private static final class MutableVersionedAccount implements VersionedAccount { + private final String password = new BCryptPasswordEncoder(12).encode("password"); + private long version = 3; + + @Override + public String getAppId() { return "owner"; } + + @Override + public String getPassword() { return password; } + + @Override + public String getSalt() { return null; } + + @Override + public List getOwnRoles() { return List.of("admin"); } + + @Override + public boolean isDisabledAccount() { return false; } + + @Override + public boolean isExcessiveAttempts() { return false; } + + @Override + public long credentialVersion() { return version; } + + @Override + public boolean bcryptPassword() { return true; } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/AccountServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/AccountServiceTest.java index 16c96f0ecb..bb58e18ced 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/AccountServiceTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/AccountServiceTest.java @@ -29,16 +29,17 @@ import io.jsonwebtoken.MalformedJwtException; import org.apache.hertzbeat.common.entity.manager.AuthToken; import org.apache.hertzbeat.common.observability.gateway.AuthTokenRequestContext; import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes; +import org.apache.hertzbeat.common.observability.gateway.ObservabilityAccessTokenGateway; import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.manager.dao.AuthTokenDao; import org.apache.hertzbeat.manager.pojo.dto.LoginDto; import org.apache.hertzbeat.manager.pojo.dto.RefreshTokenResponse; import org.apache.hertzbeat.manager.service.impl.AccountServiceImpl; +import org.apache.hertzbeat.manager.setup.identity.AccountCredentialVerifier; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; -import org.springframework.test.util.ReflectionTestUtils; import org.springframework.transaction.annotation.Transactional; import javax.naming.AuthenticationException; @@ -92,8 +93,8 @@ class AccountServiceTest { accountProvider = mock(SurenessAccountProvider.class); authTokenDao = mock(AuthTokenDao.class); - accountService = new AccountServiceImpl(accountProvider); - ReflectionTestUtils.setField(accountService, "authTokenDao", authTokenDao); + accountService = new AccountServiceImpl(accountProvider, authTokenDao, + new AccountCredentialVerifier(new org.apache.hertzbeat.manager.setup.identity.IdentityPasswordPolicy())); JsonWebTokenUtil.setDefaultSecretKey(jwt); } @@ -481,7 +482,8 @@ class AccountServiceTest { String token = accountService.generateToken("test", null); Claims claims = JsonWebTokenUtil.parseJwt(token); - assertEquals(Boolean.TRUE, claims.get(AccountServiceImpl.CLAIM_MANAGED, Boolean.class)); + assertEquals(Boolean.TRUE, + claims.get(ObservabilityAccessTokenGateway.CLAIM_MANAGED, Boolean.class)); } } @@ -770,7 +772,7 @@ class AccountServiceTest { void testCheckManagedTokenAccessValid() { when(accountProvider.loadAccount(identifier)).thenReturn(buildActiveAccount()); - String result = accountService.checkManagedTokenAccess(identifier, List.of("admin")); + String result = accountService.checkManagedTokenAccess(identifier, List.of("admin"), null); assertNull(result); } @@ -786,7 +788,7 @@ class AccountServiceTest { .build(); when(accountProvider.loadAccount(identifier)).thenReturn(account); - String result = accountService.checkManagedTokenAccess(identifier, List.of("admin")); + String result = accountService.checkManagedTokenAccess(identifier, List.of("admin"), null); assertEquals("Token owner account is no longer valid", result); } @@ -802,7 +804,7 @@ class AccountServiceTest { .build(); when(accountProvider.loadAccount(identifier)).thenReturn(account); - String result = accountService.checkManagedTokenAccess(identifier, List.of("admin")); + String result = accountService.checkManagedTokenAccess(identifier, List.of("admin"), null); assertEquals("Token permissions are outdated", result); } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/BcryptPasswordProcessorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/BcryptPasswordProcessorTest.java new file mode 100644 index 0000000000..fed7dab909 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/BcryptPasswordProcessorTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.usthe.sureness.processor.exception.IncorrectCredentialsException; +import com.usthe.sureness.provider.DefaultAccount; +import com.usthe.sureness.subject.Subject; +import com.usthe.sureness.subject.support.PasswordSubject; +import com.usthe.sureness.util.Md5Util; +import java.util.List; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +class BcryptPasswordProcessorTest { + @Test + void authenticatesPersistedBcryptCredentialOnRealPasswordSubjectPath() { + DatabaseAccountRepository repository = mock(DatabaseAccountRepository.class); + DatabaseAccount account = DatabaseAccount.firstAdministrator( + "owner", new IdentityPasswordPolicy().encode("correct".toCharArray()), "admin"); + when(repository.findByUsername("owner")).thenReturn(Optional.of(account)); + DatabaseFirstAccountProvider provider = new DatabaseFirstAccountProvider( + repository, mock(LegacyAccountSource.class)); + BcryptPasswordProcessor processor = new BcryptPasswordProcessor(provider, + new AccountCredentialVerifier(new IdentityPasswordPolicy())); + + Subject authenticated = processor.authenticated(PasswordSubject.builder("owner", "correct").build()); + + assertEquals(List.of("admin"), authenticated.getOwnRoles()); + assertThrows(IncorrectCredentialsException.class, + () -> processor.authenticated(PasswordSubject.builder("owner", "wrong").build())); + verify(repository, times(2)).findByUsername("owner"); + } + + @Test + void verifiesLegacySaltWithOneLegacyProviderLoad() { + DatabaseAccountRepository repository = mock(DatabaseAccountRepository.class); + LegacyAccountSource legacy = mock(LegacyAccountSource.class); + when(repository.findByUsername("legacy")).thenReturn(Optional.empty()); + when(legacy.loadAccount("legacy")).thenReturn( + DefaultAccount.builder("legacy") + .setPassword(Md5Util.md5("correctsalt")) + .setSalt("salt").setOwnRoles(List.of("user")).build()); + BcryptPasswordProcessor processor = new BcryptPasswordProcessor( + new DatabaseFirstAccountProvider(repository, legacy), + new AccountCredentialVerifier(new IdentityPasswordPolicy())); + + Subject authenticated = processor.authenticated(PasswordSubject.builder("legacy", "correct").build()); + + assertEquals(List.of("user"), authenticated.getOwnRoles()); + verify(legacy).loadAccount("legacy"); + } + + @Test + void verifiesSaltedLegacyCredentialWithoutConvertingSuppliedArrayToString() { + DefaultAccount legacy = DefaultAccount.builder("legacy") + .setPassword(Md5Util.md5("correctsalt")) + .setSalt("salt") + .build(); + + boolean matches = new AccountCredentialVerifier(new IdentityPasswordPolicy()) + .matches(legacy, "correct".toCharArray()); + + assertTrue(matches); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/DatabaseFirstAccountProviderTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/DatabaseFirstAccountProviderTest.java new file mode 100644 index 0000000000..83d53aa507 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/DatabaseFirstAccountProviderTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.usthe.sureness.provider.DefaultAccount; +import com.usthe.sureness.provider.SurenessAccount; +import java.util.List; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +class DatabaseFirstAccountProviderTest { + @Test + void customLegacyIdentityRemainsAvailableUntilItIsMigrated() { + DatabaseAccountRepository repository = mock(DatabaseAccountRepository.class); + LegacyAccountSource legacy = mock(LegacyAccountSource.class); + SurenessAccount legacyAccount = DefaultAccount.builder("custom").setPassword("custom-secret").build(); + when(repository.findByUsername("custom")).thenReturn(Optional.empty()); + when(legacy.loadAccount("custom")).thenReturn(legacyAccount); + DatabaseFirstAccountProvider provider = new DatabaseFirstAccountProvider(repository, legacy); + + assertSame(legacyAccount, provider.loadAccount("custom")); + + verify(legacy).loadAccount("custom"); + } + + @Test + void persistedIdentityAlwaysTakesPriority() { + DatabaseAccountRepository repository = mock(DatabaseAccountRepository.class); + LegacyAccountSource legacy = mock(LegacyAccountSource.class); + DatabaseAccount database = new DatabaseAccount("operator", "hash", "admin,user", 7, (short) 1); + when(repository.findByUsername("operator")).thenReturn(Optional.of(database)); + + SurenessAccount result = new DatabaseFirstAccountProvider(repository, legacy).loadAccount("operator"); + + assertEquals("operator", result.getAppId()); + assertEquals(List.of("admin", "user"), result.getOwnRoles()); + assertEquals(7, ((VersionedAccount) result).credentialVersion()); + verify(legacy, never()).loadAccount("operator"); + } + + @Test + void digestAuthenticationNeverReceivesBcryptDatabaseCredential() { + DatabaseAccountRepository repository = mock(DatabaseAccountRepository.class); + LegacyAccountSource legacy = mock(LegacyAccountSource.class); + DatabaseAccount database = new DatabaseAccount("operator", "bcrypt-hash", "admin", 1, (short) 1); + when(repository.findByUsername("operator")).thenReturn(Optional.of(database)); + + SurenessAccount result = new DatabaseFirstAccountProvider(repository, legacy) + .loadLegacyAccountForDigest("operator"); + + assertNull(result); + verify(legacy, never()).loadAccount("operator"); + } + + @Test + void legacyFixedDefaultCannotEnterNormalAuthentication() { + DatabaseAccountRepository repository = mock(DatabaseAccountRepository.class); + LegacyAccountSource legacy = mock(LegacyAccountSource.class); + when(repository.findByUsername("admin")).thenReturn(Optional.empty()); + when(legacy.loadAccount("admin")).thenReturn( + DefaultAccount.builder("admin").setPassword("hertzbeat").setOwnRoles(List.of("admin")).build()); + + SurenessAccount result = new DatabaseFirstAccountProvider(repository, legacy).loadAccount("admin"); + + assertTrue(result.isDisabledAccount()); + assertFalse(result.toString().contains("hertzbeat")); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/DatabaseIdentityProcessorConfigurationTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/DatabaseIdentityProcessorConfigurationTest.java new file mode 100644 index 0000000000..0fd89251df --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/DatabaseIdentityProcessorConfigurationTest.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.usthe.sureness.configuration.SurenessProperties; +import com.usthe.sureness.configuration.SurenessProperties.AuthType; +import java.util.Set; +import org.junit.jupiter.api.Test; + +class DatabaseIdentityProcessorConfigurationTest { + @Test + void nullAndEmptyAuthConfigurationRetainStarterDefaults() { + SurenessProperties properties = new SurenessProperties(); + properties.setAuths(null); + assertEquals(Set.of(AuthType.BASIC, AuthType.JWT), + DatabaseIdentityProcessorConfiguration.authTypes(properties)); + properties.setAuths(new AuthType[0]); + assertEquals(Set.of(AuthType.BASIC, AuthType.JWT), + DatabaseIdentityProcessorConfiguration.authTypes(properties)); + } + + @Test + void explicitAuthConfigurationIsPreservedExactly() { + SurenessProperties properties = new SurenessProperties(); + properties.setAuths(new AuthType[] {AuthType.DIGEST, AuthType.JWT}); + assertEquals(Set.of(AuthType.DIGEST, AuthType.JWT), + DatabaseIdentityProcessorConfiguration.authTypes(properties)); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/DatabaseIdentitySpringWiringTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/DatabaseIdentitySpringWiringTest.java new file mode 100644 index 0000000000..6fe66b637a --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/DatabaseIdentitySpringWiringTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.usthe.sureness.util.JsonWebTokenUtil; +import java.util.Optional; +import org.apache.hertzbeat.manager.dao.AuthTokenDao; +import org.apache.hertzbeat.manager.pojo.dto.LoginDto; +import org.apache.hertzbeat.manager.service.AccountService; +import org.apache.hertzbeat.manager.service.impl.AccountServiceImpl; +import org.junit.jupiter.api.Test; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; + +class DatabaseIdentitySpringWiringTest { + @Test + void accountServiceReceivesDatabaseFirstProviderFromSpring() throws Exception { + DatabaseAccountRepository repository = mock(DatabaseAccountRepository.class); + DatabaseAccount account = new DatabaseAccount("owner", + new BCryptPasswordEncoder(12).encode("correct"), "admin", 1, (short) 1); + when(repository.findByUsername("owner")).thenReturn(Optional.of(account)); + JsonWebTokenUtil.setDefaultSecretKey("long-test-key-which-is-not-a-production-secret-1234567890"); + try (AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext()) { + context.registerBean(DatabaseAccountRepository.class, () -> repository); + context.registerBean(AuthTokenDao.class, () -> mock(AuthTokenDao.class)); + context.registerBean(LegacyAccountSource.class, () -> mock(LegacyAccountSource.class)); + context.register(DatabaseFirstAccountProvider.class, AccountCredentialVerifier.class, + IdentityPasswordPolicy.class, AccountServiceImpl.class); + context.refresh(); + + AccountService service = context.getBean(AccountService.class); + assertNotNull(service.authGetToken( + LoginDto.builder().identifier("owner").credential("correct").build()).get("token")); + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/IdentityInitializationServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/IdentityInitializationServiceTest.java new file mode 100644 index 0000000000..23b4d22fd8 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/IdentityInitializationServiceTest.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Optional; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; + +class IdentityInitializationServiceTest { + @Test + void createsUniqueFirstAdministratorWithCostTwelveHash() { + DatabaseAccountRepository repository = mock(DatabaseAccountRepository.class); + CredentialRevocation revocation = mock(CredentialRevocation.class); + when(repository.existsByUsername("owner")).thenReturn(false); + IdentityInitializationService service = new IdentityInitializationService( + repository, revocation, new IdentityPasswordPolicy()); + + service.createFirstAdministrator(new AdministratorCredentials(" owner ", "correct horse".toCharArray())); + + ArgumentCaptor saved = ArgumentCaptor.forClass(DatabaseAccount.class); + verify(repository).saveAndFlush(saved.capture()); + assertEquals("owner", saved.getValue().username()); + assertTrue(saved.getValue().passwordHash().startsWith("$2a$12$")); + assertTrue(new BCryptPasswordEncoder().matches("correct horse", saved.getValue().passwordHash())); + assertFalse(saved.getValue().toString().contains("correct horse")); + } + + @Test + void refusesSecondAdministrator() { + DatabaseAccountRepository repository = mock(DatabaseAccountRepository.class); + when(repository.existsByBootstrapSlotIsNotNull()).thenReturn(true); + IdentityInitializationService service = new IdentityInitializationService( + repository, mock(CredentialRevocation.class), new IdentityPasswordPolicy()); + AdministratorCredentials credentials = new AdministratorCredentials("other", "secret".toCharArray()); + + assertThrows(IllegalStateException.class, + () -> service.createFirstAdministrator(credentials)); + assertTrue(allZero(credentials.copyPassword())); + verify(repository, never()).saveAndFlush(any()); + } + + @Test + void createsBootstrapAdministratorAfterAnOrdinaryLegacyIdentityMigrates() { + DatabaseAccountRepository repository = mock(DatabaseAccountRepository.class); + when(repository.existsByBootstrapSlotIsNotNull()).thenReturn(false); + when(repository.existsByUsername("owner")).thenReturn(false); + IdentityInitializationService service = new IdentityInitializationService( + repository, mock(CredentialRevocation.class), new IdentityPasswordPolicy()); + + service.createFirstAdministrator(new AdministratorCredentials("owner", "secret".toCharArray())); + + ArgumentCaptor saved = ArgumentCaptor.forClass(DatabaseAccount.class); + verify(repository).saveAndFlush(saved.capture()); + assertTrue(saved.getValue().bootstrapAdministrator()); + } + + @Test + void missingAccountStillClearsReplacementPassword() { + DatabaseAccountRepository repository = mock(DatabaseAccountRepository.class); + when(repository.findByUsernameForUpdate("missing")).thenReturn(Optional.empty()); + IdentityInitializationService service = new IdentityInitializationService( + repository, mock(CredentialRevocation.class), new IdentityPasswordPolicy()); + char[] replacement = "new-secret".toCharArray(); + + assertThrows(IllegalArgumentException.class, () -> service.changePassword("missing", replacement)); + + assertTrue(allZero(replacement)); + } + + @Test + void revokesOnlyAfterCredentialVersionIsPersisted() { + DatabaseAccountRepository repository = mock(DatabaseAccountRepository.class); + CredentialRevocation revocation = mock(CredentialRevocation.class); + DatabaseAccount account = new DatabaseAccount("owner", "old", "admin", 3, (short) 1); + when(repository.findByUsernameForUpdate("owner")).thenReturn(Optional.of(account)); + when(repository.save(account)).thenThrow(new IllegalStateException("storage unavailable")); + IdentityInitializationService service = new IdentityInitializationService( + repository, revocation, new IdentityPasswordPolicy()); + + char[] replacement = "new-secret".toCharArray(); + assertThrows(IllegalStateException.class, () -> service.changePassword("owner", replacement)); + assertEquals(4, account.credentialVersion()); + assertTrue(allZero(replacement)); + verify(revocation, never()).revokeFor("owner"); + } + + @Test + void mapsConcurrentBootstrapConstraintToStableConflict() { + DatabaseAccountRepository repository = mock(DatabaseAccountRepository.class); + when(repository.existsByUsername("owner")).thenReturn(false); + when(repository.saveAndFlush(any())).thenThrow(new DataIntegrityViolationException("duplicate", + new java.sql.SQLException("unique", "23505"))); + IdentityInitializationService service = new IdentityInitializationService(repository, + mock(CredentialRevocation.class), new IdentityPasswordPolicy()); + + assertThrows(BootstrapIdentityConflict.class, + () -> service.createFirstAdministrator( + new AdministratorCredentials("owner", "secret".toCharArray()))); + } + + private static boolean allZero(char[] value) { + for (char item : value) { + if (item != '\0') { + return false; + } + } + return true; + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/LegacyAccountMigrationServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/LegacyAccountMigrationServiceTest.java new file mode 100644 index 0000000000..4cc0638cce --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/LegacyAccountMigrationServiceTest.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.usthe.sureness.provider.DefaultAccount; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; + +class LegacyAccountMigrationServiceTest { + @Test + void explicitlyMigratesFixedDefaultBeforeItCanUseNormalLogin() { + Fixture fixture = fixture("admin", "hertzbeat"); + char[] oldPassword = "hertzbeat".toCharArray(); + char[] replacement = "new-owner-secret".toCharArray(); + + fixture.service.migrate("admin", oldPassword, replacement); + + ArgumentCaptor saved = ArgumentCaptor.forClass(DatabaseAccount.class); + verify(fixture.accounts).saveAndFlush(saved.capture()); + assertTrue(new BCryptPasswordEncoder().matches("new-owner-secret", saved.getValue().passwordHash())); + verify(fixture.revocation).revokeFor("admin"); + assertTrue(allZero(oldPassword)); + assertTrue(allZero(replacement)); + } + + @Test + void explicitlyMigratesCustomLegacyIdentity() { + Fixture fixture = fixture("operator", "custom-secret"); + + fixture.service.migrate("operator", "custom-secret".toCharArray(), "replacement".toCharArray()); + + verify(fixture.accounts).saveAndFlush(any(DatabaseAccount.class)); + verify(fixture.revocation).revokeFor("operator"); + } + + @Test + void customLegacyIdentityCanMigrateAfterTheFirstDatabaseAdministrator() { + Fixture fixture = fixture("operator", "custom-secret"); + when(fixture.accounts.existsByBootstrapSlotIsNotNull()).thenReturn(true); + + fixture.service.migrate("operator", "custom-secret".toCharArray(), "replacement".toCharArray()); + + verify(fixture.accounts).saveAndFlush(any(DatabaseAccount.class)); + verify(fixture.revocation).revokeFor("operator"); + } + + @Test + void nonAdministratorMigrationDoesNotConsumeTheBootstrapAdministratorSlot() { + Fixture fixture = fixture("operator", "custom-secret", List.of("guest")); + + fixture.service.migrate("operator", "custom-secret".toCharArray(), "replacement".toCharArray()); + + ArgumentCaptor saved = ArgumentCaptor.forClass(DatabaseAccount.class); + verify(fixture.accounts).saveAndFlush(saved.capture()); + assertTrue(saved.getValue().roleList().contains("guest")); + assertFalse(saved.getValue().bootstrapAdministrator()); + verify(fixture.accounts, never()).existsByBootstrapSlotIsNotNull(); + } + + @Test + void storageFailureDoesNotClaimRevocationOrMigration() { + Fixture fixture = fixture("operator", "custom-secret"); + when(fixture.accounts.saveAndFlush(any())).thenThrow(new IllegalStateException("storage unavailable")); + + assertThrows(IllegalStateException.class, + () -> fixture.service.migrate("operator", "custom-secret".toCharArray(), "replacement".toCharArray())); + verify(fixture.revocation, never()).revokeFor("operator"); + } + + @Test + void existingDatabaseIdentityStillClearsCallerPasswords() { + Fixture fixture = fixture("operator", "custom-secret"); + when(fixture.accounts.existsByUsername("operator")).thenReturn(true); + char[] legacy = "custom-secret".toCharArray(); + char[] replacement = "replacement".toCharArray(); + + assertThrows(BootstrapIdentityConflict.class, + () -> fixture.service.migrate("operator", legacy, replacement)); + + assertTrue(allZero(legacy)); + assertTrue(allZero(replacement)); + verify(fixture.accounts, never()).saveAndFlush(any()); + } + + private static Fixture fixture(String username, String password) { + return fixture(username, password, List.of("admin")); + } + + private static Fixture fixture(String username, String password, List roles) { + DatabaseAccountRepository accounts = mock(DatabaseAccountRepository.class); + LegacyAccountSource legacy = mock(LegacyAccountSource.class); + CredentialRevocation revocation = mock(CredentialRevocation.class); + IdentityPasswordPolicy passwords = new IdentityPasswordPolicy(); + when(legacy.loadAccount(username)).thenReturn(DefaultAccount.builder(username).setPassword(password) + .setOwnRoles(roles).build()); + return new Fixture(accounts, revocation, new LegacyAccountMigrationService(accounts, legacy, + new AccountCredentialVerifier(passwords), revocation, passwords)); + } + + private static boolean allZero(char[] value) { + for (char item : value) { + if (item != '\0') { + return false; + } + } + return true; + } + + private record Fixture(DatabaseAccountRepository accounts, CredentialRevocation revocation, + LegacyAccountMigrationService service) { + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/VersionedJwtProcessorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/VersionedJwtProcessorTest.java new file mode 100644 index 0000000000..e0b4cee6ec --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/VersionedJwtProcessorTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.usthe.sureness.processor.exception.ExpiredCredentialsException; +import com.usthe.sureness.provider.DefaultAccount; +import com.usthe.sureness.provider.SurenessAccount; +import com.usthe.sureness.subject.support.JwtSubject; +import com.usthe.sureness.util.JsonWebTokenUtil; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes; +import org.junit.jupiter.api.Test; + +class VersionedJwtProcessorTest { + @Test + void directSurenessPathRejectsUiSessionFromOldCredentialGeneration() { + JsonWebTokenUtil.setDefaultSecretKey("long-test-key-which-is-not-a-production-secret-1234567890"); + DatabaseAccountRepository repository = mock(DatabaseAccountRepository.class); + IdentityPasswordPolicy passwords = new IdentityPasswordPolicy(); + DatabaseAccount account = DatabaseAccount.firstAdministrator( + "owner", passwords.encode("password".toCharArray()), "admin"); + account.replacePassword(passwords.encode("replacement".toCharArray())); + when(repository.findByUsername("owner")).thenReturn(Optional.of(account)); + DatabaseFirstAccountProvider provider = new DatabaseFirstAccountProvider( + repository, mock(LegacyAccountSource.class)); + String token = JsonWebTokenUtil.issueJwt("owner", 3600L, List.of("admin"), new HashMap<>(Map.of( + AuthTokenScopes.CLAIM_TOKEN_SCOPE, AuthTokenScopes.UI_SESSION, + "credentialVersion", 1L))); + + assertThrows(ExpiredCredentialsException.class, + () -> new VersionedJwtProcessor(provider).authenticated(JwtSubject.builder(token).build())); + } + + @Test + void rejectsSessionWhenCurrentAccountIsMissing() { + assertUnavailableAccount(null); + } + + @Test + void rejectsSessionWhenCurrentAccountIsDisabled() { + SurenessAccount disabled = DefaultAccount.builder("owner").setPassword("unused") + .setDisabledAccount(true).build(); + assertUnavailableAccount(disabled); + } + + private static void assertUnavailableAccount(SurenessAccount account) { + JsonWebTokenUtil.setDefaultSecretKey("long-test-key-which-is-not-a-production-secret-1234567890"); + DatabaseAccountRepository repository = mock(DatabaseAccountRepository.class); + LegacyAccountSource legacy = mock(LegacyAccountSource.class); + when(repository.findByUsername("owner")).thenReturn(Optional.empty()); + when(repository.count()).thenReturn(0L); + when(legacy.loadAccount("owner")).thenReturn(account); + DatabaseFirstAccountProvider provider = new DatabaseFirstAccountProvider(repository, legacy); + String token = JsonWebTokenUtil.issueJwt("owner", 3600L, List.of("admin"), new HashMap<>(Map.of( + AuthTokenScopes.CLAIM_TOKEN_SCOPE, AuthTokenScopes.UI_SESSION, + "credentialVersion", 3L))); + assertThrows(ExpiredCredentialsException.class, + () -> new VersionedJwtProcessor(provider).authenticated(JwtSubject.builder(token).build())); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/installation/InstallationClassifierTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/installation/InstallationClassifierTest.java new file mode 100644 index 0000000000..66f2a72f6c --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/installation/InstallationClassifierTest.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.installation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +class InstallationClassifierTest { + private static final InstallationFingerprint LOCAL = new InstallationFingerprint("a".repeat(64)); + private final InstallationClassifier classifier = new InstallationClassifier(); + + @Test + void distinguishesSetupUpgradeFullAndFailClosedRecovery() { + assertEquals(InstallationMode.SETUP, + classifier.classify(DatabasePresence.EMPTY, Optional.empty(), Optional.empty())); + assertEquals(InstallationMode.RECOVERY, + classifier.classify(DatabasePresence.EMPTY, Optional.empty(), Optional.of(LOCAL))); + assertEquals(InstallationMode.UPGRADE, + classifier.classify(DatabasePresence.HERTZBEAT_SCHEMA, Optional.empty(), Optional.of(LOCAL))); + assertEquals(InstallationMode.FULL, + classifier.classify(DatabasePresence.HERTZBEAT_SCHEMA, + Optional.of(new InstallationRecord(LOCAL.value())), Optional.of(LOCAL))); + assertEquals(InstallationMode.RECOVERY, + classifier.classify(DatabasePresence.UNREACHABLE, Optional.empty(), Optional.empty())); + assertEquals(InstallationMode.RECOVERY, + classifier.classify(DatabasePresence.HERTZBEAT_SCHEMA, + Optional.of(new InstallationRecord("b".repeat(64))), Optional.of(LOCAL))); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/installation/InstallationPersistenceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/installation/InstallationPersistenceTest.java new file mode 100644 index 0000000000..29ea4c13d7 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/installation/InstallationPersistenceTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.installation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; +import java.security.SecureRandom; +import java.util.Optional; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.dao.DataIntegrityViolationException; + +class InstallationPersistenceTest { + private static final InstallationFingerprint FIRST = new InstallationFingerprint("a".repeat(64)); + + @TempDir + Path temporaryDirectory; + + @Test + void completionIsIdempotentOnlyForSameFingerprint() { + InstallationRecordRepository records = mock(InstallationRecordRepository.class); + when(records.findById(InstallationRecord.SINGLETON_ID)).thenReturn(Optional.empty()); + InstallationCompletionService service = new InstallationCompletionService(records); + service.complete(FIRST); + verify(records).saveAndFlush(any(InstallationRecord.class)); + + when(records.findById(InstallationRecord.SINGLETON_ID)).thenReturn(Optional.of(new InstallationRecord(FIRST.value()))); + service.complete(FIRST); + assertThrows(IllegalStateException.class, + () -> service.complete(new InstallationFingerprint("b".repeat(64)))); + } + + @Test + void failedCompletionNeverReportsWritesClosed() { + InstallationRecordRepository records = mock(InstallationRecordRepository.class); + when(records.findById(InstallationRecord.SINGLETON_ID)).thenReturn(Optional.empty()); + when(records.saveAndFlush(any())).thenThrow(new IllegalStateException("storage unavailable")); + InstallationCompletionService service = new InstallationCompletionService(records); + assertThrows(IllegalStateException.class, () -> service.complete(FIRST)); + when(records.existsById(InstallationRecord.SINGLETON_ID)).thenReturn(false); + assertFalse(service.writesClosed()); + } + + @Test + void concurrentCompletionIsIdempotentForTheSameFingerprint() { + InstallationRecordRepository records = mock(InstallationRecordRepository.class); + when(records.findById(InstallationRecord.SINGLETON_ID)) + .thenReturn(Optional.empty(), Optional.of(new InstallationRecord(FIRST.value()))); + when(records.saveAndFlush(any())).thenThrow(new DataIntegrityViolationException("concurrent insert")); + + new InstallationCompletionService(records).complete(FIRST); + + verify(records, times(2)).findById(InstallationRecord.SINGLETON_ID); + } + + @Test + void concurrentCompletionRejectsDifferentFingerprint() { + InstallationRecordRepository records = mock(InstallationRecordRepository.class); + when(records.findById(InstallationRecord.SINGLETON_ID)) + .thenReturn(Optional.empty(), Optional.of(new InstallationRecord("b".repeat(64)))); + when(records.saveAndFlush(any())).thenThrow(new DataIntegrityViolationException("concurrent insert")); + + InstallationCompletionService service = new InstallationCompletionService(records); + + assertThrows(IllegalStateException.class, () -> service.complete(FIRST)); + } + + @Test + void localFingerprintIsOwnerOnlyReadableAndCollisionSafe() throws Exception { + Path path = temporaryDirectory.resolve("installation-id"); + LocalInstallationFingerprintStore store = new LocalInstallationFingerprintStore(path, new SecureRandom()); + InstallationFingerprint created = store.create(); + assertEquals(Optional.of(created), store.read()); + assertEquals(PosixFilePermissions.fromString("rw-------"), Files.getPosixFilePermissions(path)); + assertThrows(java.nio.file.FileAlreadyExistsException.class, store::create); + } + + @Test + void localFingerprintDoesNotFollowTargetSymlink() throws Exception { + Path target = temporaryDirectory.resolve("target"); + Files.writeString(target, FIRST.value()); + Path link = temporaryDirectory.resolve("installation-id"); + Files.createSymbolicLink(link, target); + LocalInstallationFingerprintStore store = new LocalInstallationFingerprintStore(link, new SecureRandom()); + assertEquals(Optional.empty(), store.read()); + assertThrows(java.nio.file.FileAlreadyExistsException.class, store::create); + } + + @Test + void localFingerprintRejectsPermissionsThatExposeItToOtherUsers() throws Exception { + Path path = temporaryDirectory.resolve("installation-id"); + LocalInstallationFingerprintStore store = new LocalInstallationFingerprintStore(path, new SecureRandom()); + store.create(); + Files.setPosixFilePermissions(path, PosixFilePermissions.fromString("rw-r--r--")); + + assertEquals(Optional.empty(), store.read()); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/RemoteSetupUnlockTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/RemoteSetupUnlockTest.java new file mode 100644 index 0000000000..51898d1ff3 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/RemoteSetupUnlockTest.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.security; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.net.InetAddress; +import java.nio.file.DirectoryNotEmptyException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; +import java.security.SecureRandom; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.Arrays; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class RemoteSetupUnlockTest { + @TempDir + Path temporaryDirectory; + + @Test + void loopbackNeedsNoUnlockWhileRemoteBindDoes() throws Exception { + RemoteSetupUnlock unlock = unlock(temporaryDirectory.resolve("unlock")); + assertFalse(unlock.requiresUnlock(InetAddress.getLoopbackAddress())); + assertTrue(unlock.requiresUnlock(InetAddress.getByName("0.0.0.0"))); + } + + @Test + void ownerOnlyCodeIsSingleUseAndBecomesStrictHttpOnlyCookie() throws Exception { + Path codeFile = temporaryDirectory.resolve("unlock"); + Clock clock = Clock.fixed(Instant.parse("2026-08-08T00:00:00Z"), ZoneOffset.UTC); + RemoteSetupUnlock unlock = new RemoteSetupUnlock(codeFile, clock, deterministicRandom()); + unlock.open(); + String code = Files.readString(codeFile); + assertTrue(Files.getPosixFilePermissions(codeFile) + .equals(PosixFilePermissions.fromString("rw-------"))); + + SetupAccessSession session = unlock.redeem("198.51.100.4", new SetupUnlockCode(code.toCharArray())); + + assertFalse(Files.exists(codeFile)); + assertTrue(unlock.permits(session.token())); + SetupUnlockRejected reused = assertThrows(SetupUnlockRejected.class, + () -> unlock.redeem("198.51.100.4", new SetupUnlockCode(code.toCharArray()))); + assertEquals(SetupUnlockRejected.Reason.INVALID, reused.reason()); + String cookie = SetupAccessCookie.create(session, true, clock).toString(); + assertTrue(cookie.contains("HttpOnly")); + assertTrue(cookie.contains("SameSite=Strict")); + assertTrue(cookie.contains("Secure")); + assertFalse(session.toString().contains(session.token())); + } + + @Test + void limitsRepeatedInvalidProofs() throws Exception { + RemoteSetupUnlock unlock = unlock(temporaryDirectory.resolve("unlock")); + unlock.open(); + for (int attempt = 0; attempt < 5; attempt++) { + assertThrows(SetupUnlockRejected.class, + () -> unlock.redeem("203.0.113.8", new SetupUnlockCode("wrong".toCharArray()))); + } + SetupUnlockRejected limited = assertThrows(SetupUnlockRejected.class, + () -> unlock.redeem("203.0.113.8", new SetupUnlockCode("wrong".toCharArray()))); + assertEquals(SetupUnlockRejected.Reason.RATE_LIMITED, limited.reason()); + } + + @Test + void distinguishesAnExpiredProofFromAnInvalidProof() throws Exception { + Instant openedAt = Instant.parse("2026-08-08T00:00:00Z"); + Clock clock = mock(Clock.class); + when(clock.instant()).thenReturn(openedAt, openedAt.plus(Duration.ofMinutes(16))); + Path codeFile = temporaryDirectory.resolve("unlock"); + RemoteSetupUnlock unlock = new RemoteSetupUnlock(codeFile, clock, deterministicRandom()); + unlock.open(); + + SetupUnlockRejected expired = assertThrows(SetupUnlockRejected.class, + () -> unlock.redeem( + "198.51.100.4", new SetupUnlockCode(Files.readString(codeFile).toCharArray()))); + + assertEquals(SetupUnlockRejected.Reason.EXPIRED, expired.reason()); + } + + @Test + void restartRotatesStaleOwnerOnlyUnlockFile() throws Exception { + Path codeFile = temporaryDirectory.resolve("unlock"); + Clock clock = Clock.fixed(Instant.parse("2026-08-08T00:00:00Z"), ZoneOffset.UTC); + SecureRandom random = deterministicRandom(); + RemoteSetupUnlock firstProcess = new RemoteSetupUnlock(codeFile, clock, random); + firstProcess.open(); + String staleCode = Files.readString(codeFile); + + RemoteSetupUnlock restarted = new RemoteSetupUnlock(codeFile, clock, random); + restarted.open(); + String replacementCode = Files.readString(codeFile); + + assertNotEquals(staleCode, replacementCode); + assertThrows(SetupUnlockRejected.class, + () -> restarted.redeem("198.51.100.4", new SetupUnlockCode(staleCode.toCharArray()))); + SetupAccessSession session = restarted.redeem( + "198.51.100.4", new SetupUnlockCode(replacementCode.toCharArray())); + assertTrue(restarted.permits(session.token())); + } + + @Test + void openingNewProofInvalidatesPreviousSession() throws Exception { + Path codeFile = temporaryDirectory.resolve("unlock"); + Clock clock = Clock.fixed(Instant.parse("2026-08-08T00:00:00Z"), ZoneOffset.UTC); + RemoteSetupUnlock unlock = new RemoteSetupUnlock(codeFile, clock, deterministicRandom()); + unlock.open(); + SetupAccessSession oldSession = unlock.redeem( + "198.51.100.4", new SetupUnlockCode(Files.readString(codeFile).toCharArray())); + + unlock.open(); + + assertFalse(unlock.permits(oldSession.token())); + } + + @Test + void failedProofRemovalDoesNotPublishUnreachableSession() throws Exception { + Path codeFile = temporaryDirectory.resolve("unlock"); + RemoteSetupUnlock unlock = unlock(codeFile); + unlock.open(); + String code = Files.readString(codeFile); + Files.delete(codeFile); + Files.createDirectory(codeFile); + Files.writeString(codeFile.resolve("blocker"), "keep-directory-non-empty"); + + assertThrows(DirectoryNotEmptyException.class, + () -> unlock.redeem("198.51.100.4", new SetupUnlockCode(code.toCharArray()))); + + Files.delete(codeFile.resolve("blocker")); + Files.delete(codeFile); + SetupAccessSession recovered = unlock.redeem( + "198.51.100.4", new SetupUnlockCode(code.toCharArray())); + assertTrue(unlock.permits(recovered.token())); + } + + private static RemoteSetupUnlock unlock(Path path) { + return new RemoteSetupUnlock(path, + Clock.fixed(Instant.parse("2026-08-08T00:00:00Z"), ZoneOffset.UTC), deterministicRandom()); + } + + private static SecureRandom deterministicRandom() { + return new SecureRandom() { + private byte next = 1; + + @Override + public void nextBytes(byte[] bytes) { + Arrays.fill(bytes, next++); + } + }; + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/ui/session/UiSessionServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/ui/session/UiSessionServiceTest.java index 09d20f56e2..8c3837fa87 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/ui/session/UiSessionServiceTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/ui/session/UiSessionServiceTest.java @@ -67,7 +67,7 @@ class UiSessionServiceTest { "token", accessToken, "refreshToken", refreshToken, "role", "[\"admin\"]")); - when(accountService.checkManagedTokenAccess("admin", List.of("admin"))).thenReturn(null); + when(accountService.checkSessionAccess("admin", List.of("admin"), null)).thenReturn(null); UiSessionTokens result = service.login(login); @@ -91,7 +91,7 @@ class UiSessionServiceTest { assertEquals(UiSessionView.anonymous(), service.inspect(wrongScope)); String inactiveAccount = accessToken("admin", List.of("admin"), "team-a", 3600); - when(accountService.checkManagedTokenAccess("admin", List.of("admin"))).thenReturn("disabled"); + when(accountService.checkSessionAccess("admin", List.of("admin"), null)).thenReturn("disabled"); assertEquals(UiSessionView.anonymous(), service.inspect(inactiveAccount)); } @@ -101,7 +101,7 @@ class UiSessionServiceTest { String newAccess = accessToken("admin", List.of("admin"), null, 3600); String newRefresh = JsonWebTokenUtil.issueJwt("admin", 7200L, Map.of("refresh", true)); when(accountService.refreshToken(oldRefresh)).thenReturn(new RefreshTokenResponse(newAccess, newRefresh)); - when(accountService.checkManagedTokenAccess("admin", List.of("admin"))).thenReturn(null); + when(accountService.checkSessionAccess("admin", List.of("admin"), null)).thenReturn(null); UiSessionTokens result = service.refresh(oldRefresh); diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/config/OtlpGrpcServerConfig.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/config/OtlpGrpcServerConfig.java index 285680928a..6b1a75ba41 100644 --- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/config/OtlpGrpcServerConfig.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/config/OtlpGrpcServerConfig.java @@ -150,7 +150,8 @@ public class OtlpGrpcServerConfig { List claimedRoles = claims.get("roles", List.class); rejectReason = accessTokenGateway.checkManagedTokenAccess( userId, - claimedRoles == null ? Collections.emptyList() : claimedRoles + claimedRoles == null ? Collections.emptyList() : claimedRoles, + claims.get(ObservabilityAccessTokenGateway.CLAIM_CREDENTIAL_VERSION, Long.class) ); if (rejectReason != null) { call.close(Status.UNAUTHENTICATED.withDescription(rejectReason), new Metadata()); diff --git a/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/ingestion/config/OtlpGrpcServerConfigTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/ingestion/config/OtlpGrpcServerConfigTest.java index 8152bdbcb0..3cc848eb4a 100644 --- a/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/ingestion/config/OtlpGrpcServerConfigTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/ingestion/config/OtlpGrpcServerConfigTest.java @@ -79,7 +79,7 @@ class OtlpGrpcServerConfigTest { String token = issueManagedToken(); Metadata headers = bearerHeaders(token); when(accessTokenGateway.checkTokenStatus(token, AuthTokenScopes.OTLP_INGEST)).thenReturn(null); - when(accessTokenGateway.checkManagedTokenAccess("admin", List.of("admin"))).thenReturn(null); + when(accessTokenGateway.checkManagedTokenAccess("admin", List.of("admin"), 3L)).thenReturn(null); when(next.startCall(any(), any())).thenReturn(new ServerCall.Listener<>() { }); @@ -87,7 +87,7 @@ class OtlpGrpcServerConfigTest { verify(next).startCall(call, headers); verify(accessTokenGateway).checkTokenStatus(token, AuthTokenScopes.OTLP_INGEST); - verify(accessTokenGateway).checkManagedTokenAccess("admin", List.of("admin")); + verify(accessTokenGateway).checkManagedTokenAccess("admin", List.of("admin"), 3L); verify(accessTokenGateway).touchTokenLastUsedTime(token); } @@ -97,7 +97,7 @@ class OtlpGrpcServerConfigTest { Metadata headers = bearerHeaders(token); headers.put(WORKSPACE_ID, "prod-west"); when(accessTokenGateway.checkTokenStatus(token, AuthTokenScopes.OTLP_INGEST, "prod-west")).thenReturn(null); - when(accessTokenGateway.checkManagedTokenAccess("admin", List.of("admin"))).thenReturn(null); + when(accessTokenGateway.checkManagedTokenAccess("admin", List.of("admin"), 3L)).thenReturn(null); when(next.startCall(any(), any())).thenReturn(new ServerCall.Listener<>() { }); @@ -114,7 +114,7 @@ class OtlpGrpcServerConfigTest { Metadata headers = bearerHeaders(token); headers.put(WORKSPACE_ID, " prod-west "); when(accessTokenGateway.checkTokenStatus(token, AuthTokenScopes.OTLP_INGEST, "prod-west")).thenReturn(null); - when(accessTokenGateway.checkManagedTokenAccess("admin", List.of("admin"))).thenReturn(null); + when(accessTokenGateway.checkManagedTokenAccess("admin", List.of("admin"), 3L)).thenReturn(null); when(next.startCall(any(), any())).thenAnswer(invocation -> { assertEquals("prod-west", AuthTokenRequestContext.currentWorkspaceId()); return new ServerCall.Listener<>() { @@ -139,7 +139,7 @@ class OtlpGrpcServerConfigTest { when(methodDescriptor.getFullMethodName()) .thenReturn("opentelemetry.proto.collector.metrics.v1.MetricsService/Export"); when(accessTokenGateway.checkTokenStatus(token, AuthTokenScopes.OTLP_INGEST)).thenReturn(null); - when(accessTokenGateway.checkManagedTokenAccess("admin", List.of("admin"))).thenReturn(null); + when(accessTokenGateway.checkManagedTokenAccess("admin", List.of("admin"), 3L)).thenReturn(null); when(next.startCall(any(), any())).thenAnswer(invocation -> { assertEquals("edge-west", AuthTokenRequestContext.currentCollectorId()); return new ServerCall.Listener<>() { @@ -161,7 +161,7 @@ class OtlpGrpcServerConfigTest { when(methodDescriptor.getFullMethodName()) .thenReturn("opentelemetry.proto.collector.trace.v1.TraceService/Export"); when(accessTokenGateway.checkTokenStatus(token, AuthTokenScopes.OTLP_INGEST)).thenReturn(null); - when(accessTokenGateway.checkManagedTokenAccess("admin", List.of("admin"))).thenReturn(null); + when(accessTokenGateway.checkManagedTokenAccess("admin", List.of("admin"), 3L)).thenReturn(null); interceptor.interceptCall(call, headers, next); @@ -210,7 +210,7 @@ class OtlpGrpcServerConfigTest { verify(call).close(statusCaptor.capture(), any(Metadata.class)); verify(next, never()).startCall(any(), any()); verify(accessTokenGateway).checkTokenStatus(token, AuthTokenScopes.OTLP_INGEST); - verify(accessTokenGateway, never()).checkManagedTokenAccess(any(), any()); + verify(accessTokenGateway, never()).checkManagedTokenAccess(any(), any(), any()); verify(accessTokenGateway, never()).touchTokenLastUsedTime(any()); assertEquals(Status.Code.UNAUTHENTICATED, statusCaptor.getValue().getCode()); } @@ -220,7 +220,7 @@ class OtlpGrpcServerConfigTest { String token = issueManagedToken(); Metadata headers = bearerHeaders(token); when(accessTokenGateway.checkTokenStatus(token, AuthTokenScopes.OTLP_INGEST)).thenReturn(null); - when(accessTokenGateway.checkManagedTokenAccess("admin", List.of("admin"))) + when(accessTokenGateway.checkManagedTokenAccess("admin", List.of("admin"), 3L)) .thenReturn("Token owner account is no longer valid"); interceptor.interceptCall(call, headers, next); @@ -229,7 +229,7 @@ class OtlpGrpcServerConfigTest { verify(call).close(statusCaptor.capture(), any(Metadata.class)); verify(next, never()).startCall(any(), any()); verify(accessTokenGateway).checkTokenStatus(token, AuthTokenScopes.OTLP_INGEST); - verify(accessTokenGateway).checkManagedTokenAccess("admin", List.of("admin")); + verify(accessTokenGateway).checkManagedTokenAccess("admin", List.of("admin"), 3L); verify(accessTokenGateway, never()).touchTokenLastUsedTime(any()); assertEquals(Status.Code.UNAUTHENTICATED, statusCaptor.getValue().getCode()); } @@ -241,14 +241,16 @@ class OtlpGrpcServerConfigTest { } private static String issueManagedToken() { - Map customClaims = new HashMap<>(1); + Map customClaims = new HashMap<>(2); customClaims.put(ObservabilityAccessTokenGateway.CLAIM_MANAGED, true); + customClaims.put(ObservabilityAccessTokenGateway.CLAIM_CREDENTIAL_VERSION, 3L); return JsonWebTokenUtil.issueJwt("admin", 3600L, List.of("admin"), customClaims); } private static String issueManagedCollectorToken(List allowedSignals) { Map customClaims = new HashMap<>(); customClaims.put(ObservabilityAccessTokenGateway.CLAIM_MANAGED, true); + customClaims.put(ObservabilityAccessTokenGateway.CLAIM_CREDENTIAL_VERSION, 3L); customClaims.put(AuthTokenScopes.CLAIM_TOKEN_AUDIENCE, AuthTokenScopes.MANAGED_COLLECTOR_AUDIENCE); customClaims.put(AuthTokenScopes.CLAIM_COLLECTOR_ID, "edge-west"); customClaims.put(AuthTokenScopes.CLAIM_ALLOWED_SIGNALS, allowedSignals); diff --git a/hertzbeat-startup/pom.xml b/hertzbeat-startup/pom.xml index e2e6b5deb6..997d785106 100644 --- a/hertzbeat-startup/pom.xml +++ b/hertzbeat-startup/pom.xml @@ -174,6 +174,28 @@ ${testcontainers.version} test + + org.testcontainers + testcontainers-mysql + ${testcontainers.version} + test + + + org.testcontainers + testcontainers-postgresql + ${testcontainers.version} + test + + + com.mysql + mysql-connector-j + test + + + org.postgresql + postgresql + test + diff --git a/hertzbeat-startup/src/main/resources/db/migration/h2/V205__add_identity_and_installation.sql b/hertzbeat-startup/src/main/resources/db/migration/h2/V205__add_identity_and_installation.sql new file mode 100644 index 0000000000..f5223f529c --- /dev/null +++ b/hertzbeat-startup/src/main/resources/db/migration/h2/V205__add_identity_and_installation.sql @@ -0,0 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0. +CREATE TABLE IF NOT EXISTS hzb_account ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + username VARCHAR(64) NOT NULL, + password_hash VARCHAR(100) NOT NULL, + roles VARCHAR(128) NOT NULL, + credential_version BIGINT NOT NULL, + disabled BOOLEAN NOT NULL, + bootstrap_slot SMALLINT, + CONSTRAINT uk_hzb_account_username UNIQUE (username), + CONSTRAINT uk_hzb_account_bootstrap UNIQUE (bootstrap_slot) +); +CREATE TABLE IF NOT EXISTS hzb_installation ( + id SMALLINT PRIMARY KEY, + installation_fingerprint VARCHAR(64) NOT NULL UNIQUE, + complete BOOLEAN NOT NULL +); diff --git a/hertzbeat-startup/src/main/resources/db/migration/mysql/V205__add_identity_and_installation.sql b/hertzbeat-startup/src/main/resources/db/migration/mysql/V205__add_identity_and_installation.sql new file mode 100644 index 0000000000..d73dbfeaae --- /dev/null +++ b/hertzbeat-startup/src/main/resources/db/migration/mysql/V205__add_identity_and_installation.sql @@ -0,0 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0. +CREATE TABLE IF NOT EXISTS hzb_account ( + id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, + username VARCHAR(64) NOT NULL, + password_hash VARCHAR(100) NOT NULL, + roles VARCHAR(128) NOT NULL, + credential_version BIGINT NOT NULL, + disabled BOOLEAN NOT NULL, + bootstrap_slot SMALLINT, + CONSTRAINT uk_hzb_account_username UNIQUE (username), + CONSTRAINT uk_hzb_account_bootstrap UNIQUE (bootstrap_slot) +); +CREATE TABLE IF NOT EXISTS hzb_installation ( + id SMALLINT PRIMARY KEY, + installation_fingerprint VARCHAR(64) NOT NULL UNIQUE, + complete BOOLEAN NOT NULL +); diff --git a/hertzbeat-startup/src/main/resources/db/migration/postgresql/V205__add_identity_and_installation.sql b/hertzbeat-startup/src/main/resources/db/migration/postgresql/V205__add_identity_and_installation.sql new file mode 100644 index 0000000000..f5223f529c --- /dev/null +++ b/hertzbeat-startup/src/main/resources/db/migration/postgresql/V205__add_identity_and_installation.sql @@ -0,0 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0. +CREATE TABLE IF NOT EXISTS hzb_account ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + username VARCHAR(64) NOT NULL, + password_hash VARCHAR(100) NOT NULL, + roles VARCHAR(128) NOT NULL, + credential_version BIGINT NOT NULL, + disabled BOOLEAN NOT NULL, + bootstrap_slot SMALLINT, + CONSTRAINT uk_hzb_account_username UNIQUE (username), + CONSTRAINT uk_hzb_account_bootstrap UNIQUE (bootstrap_slot) +); +CREATE TABLE IF NOT EXISTS hzb_installation ( + id SMALLINT PRIMARY KEY, + installation_fingerprint VARCHAR(64) NOT NULL UNIQUE, + complete BOOLEAN NOT NULL +); diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/identity/IdentityMigrationDatabaseTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/identity/IdentityMigrationDatabaseTest.java new file mode 100644 index 0000000000..1fc6bc3d4b --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/identity/IdentityMigrationDatabaseTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.testcontainers.mysql.MySQLContainer; +import org.testcontainers.postgresql.PostgreSQLContainer; + +/** Executes identity migrations against the two production metadata database dialects. */ +@EnabledIfSystemProperty(named = "hertzbeat.test.database-containers", matches = "true") +class IdentityMigrationDatabaseTest { + private static final String DATABASE = "hertzbeat"; + private static final String USERNAME = "hertzbeat"; + private static final String PASSWORD = "test-only-password"; + + @Test + void mysqlMigrationExecutesAgainstRealDatabase() throws Exception { + try (MySQLContainer database = new MySQLContainer("mysql:8.4") + .withDatabaseName(DATABASE) + .withUsername(USERNAME) + .withPassword(PASSWORD)) { + database.start(); + verifyMigration("mysql", database.getJdbcUrl()); + } + } + + @Test + void postgresqlMigrationExecutesAgainstRealDatabase() throws Exception { + try (PostgreSQLContainer database = new PostgreSQLContainer("postgres:17.6") + .withDatabaseName(DATABASE) + .withUsername(USERNAME) + .withPassword(PASSWORD)) { + database.start(); + verifyMigration("postgresql", database.getJdbcUrl()); + } + } + + private static void verifyMigration(String dialect, String jdbcUrl) throws Exception { + try (Connection connection = DriverManager.getConnection(jdbcUrl, USERNAME, PASSWORD)) { + IdentityMigrationResourceTest.executeMigration( + connection, IdentityMigrationResourceTest.migration(dialect)); + long accountId = insertAdministrator(connection, "owner", 1); + assertTrue(accountId > 0); + assertConstraintViolation(connection, accountInsert("owner", null)); + assertConstraintViolation(connection, accountInsert("other", 1)); + try (Statement statement = connection.createStatement()) { + statement.executeUpdate("INSERT INTO hzb_installation " + + "(id, installation_fingerprint, complete) VALUES (1, '" + "a".repeat(64) + "', TRUE)"); + } + assertEquals(1, rowCount(connection, "hzb_account")); + assertEquals(1, rowCount(connection, "hzb_installation")); + } + } + + private static long insertAdministrator(Connection connection, String username, int bootstrapSlot) + throws SQLException { + String sql = "INSERT INTO hzb_account " + + "(username, password_hash, roles, credential_version, disabled, bootstrap_slot) " + + "VALUES (?, 'hash', 'admin', 1, FALSE, ?)"; + try (PreparedStatement statement = connection.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS)) { + statement.setString(1, username); + statement.setInt(2, bootstrapSlot); + statement.executeUpdate(); + try (ResultSet keys = statement.getGeneratedKeys()) { + assertTrue(keys.next()); + return keys.getLong(1); + } + } + } + + private static String accountInsert(String username, Integer bootstrapSlot) { + String slot = bootstrapSlot == null ? "NULL" : bootstrapSlot.toString(); + return "INSERT INTO hzb_account " + + "(username, password_hash, roles, credential_version, disabled, bootstrap_slot) VALUES ('" + + username + "', 'hash', 'admin', 1, FALSE, " + slot + ")"; + } + + private static void assertConstraintViolation(Connection connection, String sql) throws SQLException { + try (Statement statement = connection.createStatement()) { + try { + statement.executeUpdate(sql); + } catch (SQLException exception) { + assertTrue(exception.getSQLState().startsWith("23")); + return; + } + } + throw new AssertionError("Expected a database constraint violation"); + } + + private static int rowCount(Connection connection, String table) throws SQLException { + try (Statement statement = connection.createStatement(); + ResultSet result = statement.executeQuery("SELECT COUNT(*) FROM " + table)) { + assertTrue(result.next()); + return result.getInt(1); + } + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/identity/IdentityMigrationResourceTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/identity/IdentityMigrationResourceTest.java new file mode 100644 index 0000000000..84613ebc94 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/identity/IdentityMigrationResourceTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; +import org.h2.jdbcx.JdbcDataSource; +import org.junit.jupiter.api.Test; + +class IdentityMigrationResourceTest { + private static final String MIGRATION = "V205__add_identity_and_installation.sql"; + private static final List REQUIRED_SCHEMA = List.of( + "username VARCHAR(64) NOT NULL", + "password_hash VARCHAR(100) NOT NULL", + "roles VARCHAR(128) NOT NULL", + "credential_version BIGINT NOT NULL", + "disabled BOOLEAN NOT NULL", + "bootstrap_slot SMALLINT", + "uk_hzb_account_username UNIQUE (username)", + "uk_hzb_account_bootstrap UNIQUE (bootstrap_slot)", + "installation_fingerprint VARCHAR(64) NOT NULL UNIQUE", + "complete BOOLEAN NOT NULL"); + + @Test + void h2MigrationExecutesAndEnforcesOneBootstrapAdministrator() throws Exception { + JdbcDataSource dataSource = new JdbcDataSource(); + dataSource.setURL("jdbc:h2:mem:identity-migration;DB_CLOSE_DELAY=-1"); + try (Connection connection = dataSource.getConnection()) { + executeMigration(connection, migration("h2")); + try (Statement statement = connection.createStatement()) { + statement.executeUpdate(accountInsert("owner", 1)); + SQLException conflict = assertThrows(SQLException.class, + () -> statement.executeUpdate(accountInsert("other", 1))); + assertEquals("23505", conflict.getSQLState()); + statement.executeUpdate("INSERT INTO hzb_installation " + + "(id, installation_fingerprint, complete) VALUES (1, '" + "a".repeat(64) + "', TRUE)"); + } + } + } + + @Test + void allDatabaseMigrationsDeclareTheSameIdentityBoundary() throws Exception { + for (String database : List.of("h2", "mysql", "postgresql")) { + String migration = migration(database); + for (String required : REQUIRED_SCHEMA) { + assertTrue(migration.contains(required), () -> database + " migration is missing: " + required); + } + } + assertTrue(migration("mysql").contains("AUTO_INCREMENT")); + assertTrue(migration("h2").contains("GENERATED BY DEFAULT AS IDENTITY")); + assertTrue(migration("postgresql").contains("GENERATED BY DEFAULT AS IDENTITY")); + } + + private static String accountInsert(String username, int bootstrapSlot) { + return "INSERT INTO hzb_account " + + "(username, password_hash, roles, credential_version, disabled, bootstrap_slot) VALUES ('" + + username + "', 'hash', 'admin', 1, FALSE, " + bootstrapSlot + ")"; + } + + static void executeMigration(Connection connection, String migration) throws SQLException { + try (Statement statement = connection.createStatement()) { + for (String sql : migration.replaceAll("(?m)^--.*$", "").split(";")) { + if (!sql.isBlank()) { + statement.execute(sql); + } + } + } + } + + static String migration(String database) throws IOException { + String resource = "/db/migration/" + database + "/" + MIGRATION; + try (InputStream stream = IdentityMigrationResourceTest.class.getResourceAsStream(resource)) { + if (stream == null) { + throw new IOException("Migration resource is missing: " + resource); + } + return new String(stream.readAllBytes(), StandardCharsets.UTF_8); + } + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/identity/PackagedLegacyDefaultAuthenticationTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/identity/PackagedLegacyDefaultAuthenticationTest.java new file mode 100644 index 0000000000..8e09bab38a --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/identity/PackagedLegacyDefaultAuthenticationTest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.usthe.sureness.processor.exception.DisabledAccountException; +import com.usthe.sureness.subject.support.PasswordSubject; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +/** Proves the fixed credential in the packaged startup resource is migration-only. */ +class PackagedLegacyDefaultAuthenticationTest { + + @Test + void packagedAdminDefaultCannotAuthenticate() { + DatabaseAccountRepository repository = mock(DatabaseAccountRepository.class); + when(repository.findByUsername("admin")).thenReturn(Optional.empty()); + when(repository.count()).thenReturn(0L); + DatabaseFirstAccountProvider provider = new DatabaseFirstAccountProvider(repository, new LegacyAccountSource()); + BcryptPasswordProcessor processor = new BcryptPasswordProcessor(provider, + new AccountCredentialVerifier(new IdentityPasswordPolicy())); + + assertThrows(DisabledAccountException.class, + () -> processor.authenticated(PasswordSubject.builder("admin", "hertzbeat").build())); + } +} From 70c9034b89ed8935f021aeceb412689cbb1f956f Mon Sep 17 00:00:00 2001 From: Logic Date: Sat, 8 Aug 2026 19:05:03 +0800 Subject: [PATCH 11/71] Complete setup workflow and validation --- .../setup/api/SetupApiConfiguration.java | 196 ++++++++++++++ .../manager/setup/api/SetupApiContract.java | 14 +- .../manager/setup/api/SetupApiException.java | 41 +++ .../manager/setup/api/SetupController.java | 137 ++++++++++ .../manager/setup/api/SetupErrorResponse.java | 30 +++ .../setup/api/SetupExceptionHandler.java | 80 ++++++ .../setup/api/SetupRuntimeStateFactory.java | 70 +++++ .../api/SetupStatusProjectionFactory.java | 77 ++++++ .../ApplicationConfigDocumentCodec.java | 109 +++++++- .../config/FileManagedSnapshotStore.java | 47 ++-- .../ManagedActiveConfigurationInspector.java | 18 +- .../config/ManagedApplicationConfig.java | 9 +- .../config/ManagedConfigurationBundle.java | 13 +- .../config/ManagedConfigurationKeys.java | 43 +++ .../config/ManagedConfigurationRecovery.java | 140 ++++++++++ .../ManagedConfigurationTransaction.java | 214 +++++---------- .../config/ManagedOptionalConfiguration.java | 80 ++++++ .../setup/config/ManagedOptionsUpdate.java | 65 +++++ .../manager/setup/config/ManagedSecrets.java | 18 +- .../config/SecretConfigDocumentCodec.java | 53 ++-- .../manager/setup/config/SecretValue.java | 24 +- .../setup/config/SensitiveExportContent.java | 20 +- .../setup/config/SetupInstallationPaths.java | 33 +++ .../identity/DatabaseAccountRepository.java | 2 + .../InstallationConvergenceService.java | 51 ++++ .../runtime/SetupResponseTransition.java | 44 +++ .../SetupResponseTransitionFilter.java | 52 ++++ .../SetupRuntimeAccessConfiguration.java | 32 +++ .../setup/runtime/SetupRuntimeTransition.java | 4 + .../SetupRuntimeTransitionScheduler.java | 99 +++++++ .../setup/security/RemoteSetupUnlock.java | 12 +- .../security/SetupHttpUnlockService.java | 119 +++++++++ .../security/SetupRequestSecurityPolicy.java | 66 +++++ .../setup/security/SetupUnlockRejected.java | 2 +- .../security/SetupWriteAccessFilter.java | 83 ++++++ .../unattended/SetupPasswordFileLoader.java | 117 ++++++++ .../UnattendedSetupInitializer.java | 160 +++++++++++ .../setup/workflow/DefaultSetupWorkflow.java | 220 +++++++++++++++ .../workflow/GreptimeHttpConnectionProbe.java | 98 +++++++ .../workflow/HeadlessSetupCoordinator.java | 114 ++++++++ .../setup/workflow/HeadlessSetupWorkflow.java | 57 ++++ .../workflow/JakartaMailConnectionProbe.java | 63 +++++ .../workflow/JdbcMetadataConnectionProbe.java | 209 +++++++++++++++ .../workflow/MailConfigurationValidator.java | 42 +++ .../setup/workflow/MailConnectionProbe.java | 28 ++ .../MetadataConfigurationValidator.java | 53 ++++ .../workflow/MetadataConnectionProbe.java | 53 ++++ .../PublicAccessConfigurationValidator.java | 75 ++++++ .../workflow/SetupCompletionCoordinator.java | 60 +++++ .../SetupConfigurationCoordinator.java | 116 ++++++++ .../workflow/SetupConfigurationMapper.java | 70 +++++ .../SetupConfigurationProjection.java | 39 +++ .../setup/workflow/SetupExportRenderer.java | 118 ++++++++ .../workflow/SetupMutationSerializer.java | 32 +++ .../workflow/SetupOperationRegistry.java | 95 +++++++ .../workflow/SetupOptionsCoordinator.java | 74 ++++++ .../setup/workflow/SetupRequestValidator.java | 99 +++++++ .../setup/workflow/SetupRuntimeState.java | 144 ++++++++++ .../setup/workflow/SetupWarningPolicy.java | 55 ++++ .../setup/workflow/SetupWorkflowConflict.java | 25 ++ .../TelemetryConfigurationValidator.java | 67 +++++ .../workflow/TelemetryConnectionProbe.java | 63 +++++ .../setup/api/SetupApiConfigurationTest.java | 36 +++ .../setup/api/SetupApiContractTest.java | 4 +- .../setup/api/SetupControllerTest.java | 158 +++++++++++ .../api/SetupRuntimeStateFactoryTest.java | 52 ++++ .../api/SetupStatusProjectionFactoryTest.java | 66 +++++ .../ManagedConfigurationTransactionTest.java | 21 ++ ...dOptionalConfigurationPersistenceTest.java | 65 +++++ .../InstallationConvergenceServiceTest.java | 73 +++++ .../SetupResponseTransitionFilterTest.java | 58 ++++ .../SetupRuntimeTransitionSchedulerTest.java | 124 +++++++++ .../setup/security/RemoteSetupUnlockTest.java | 61 +++++ .../security/SetupHttpUnlockServiceTest.java | 251 ++++++++++++++++++ .../SetupPasswordFileLoaderTest.java | 76 ++++++ .../UnattendedSetupInitializerTest.java | 120 +++++++++ .../workflow/DefaultSetupWorkflowTest.java | 207 +++++++++++++++ .../GreptimeHttpConnectionProbeTest.java | 60 +++++ .../HeadlessSetupCoordinatorTest.java | 60 +++++ .../JakartaMailConnectionProbeTest.java | 70 +++++ .../JdbcMetadataConnectionProbeTest.java | 118 ++++++++ .../SetupConfigurationCoordinatorTest.java | 123 +++++++++ .../SetupConfigurationMapperTest.java | 51 ++++ .../SetupRequestValidatorProbeTest.java | 172 ++++++++++++ .../workflow/SetupRequestValidatorTest.java | 68 +++++ .../workflow/SetupWarningPolicyTest.java | 39 +++ .../bootstrap/SetupOnlyApplication.java | 3 +- .../startup/HertzBeatApplication.java | 3 +- ...ManagedConfigEnvironmentPostProcessor.java | 3 +- .../runtime/HertzBeatStartupCoordinator.java | 6 + .../LocalInstallationStartupProbe.java | 99 +++++++ .../LocalInstallationStartupProbeTest.java | 72 +++++ 92 files changed, 6539 insertions(+), 223 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiException.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupController.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupErrorResponse.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandler.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupRuntimeStateFactory.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupStatusProjectionFactory.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationKeys.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationRecovery.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionalConfiguration.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionsUpdate.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SetupInstallationPaths.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationConvergenceService.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupResponseTransition.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupResponseTransitionFilter.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransitionScheduler.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupHttpUnlockService.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupRequestSecurityPolicy.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupWriteAccessFilter.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/unattended/SetupPasswordFileLoader.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializer.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflow.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/GreptimeHttpConnectionProbe.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinator.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupWorkflow.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JakartaMailConnectionProbe.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataConnectionProbe.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MailConfigurationValidator.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MailConnectionProbe.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataConfigurationValidator.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataConnectionProbe.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/PublicAccessConfigurationValidator.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupCompletionCoordinator.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationCoordinator.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationMapper.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationProjection.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupExportRenderer.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupMutationSerializer.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOperationRegistry.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOptionsCoordinator.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidator.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupRuntimeState.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupWarningPolicy.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupWorkflowConflict.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TelemetryConfigurationValidator.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TelemetryConnectionProbe.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfigurationTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupControllerTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupRuntimeStateFactoryTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupStatusProjectionFactoryTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionalConfigurationPersistenceTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/installation/InstallationConvergenceServiceTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupResponseTransitionFilterTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransitionSchedulerTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SetupHttpUnlockServiceTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/SetupPasswordFileLoaderTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializerTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/GreptimeHttpConnectionProbeTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinatorTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JakartaMailConnectionProbeTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataConnectionProbeTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationCoordinatorTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationMapperTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidatorProbeTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidatorTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupWarningPolicyTest.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbe.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbeTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java new file mode 100644 index 0000000000..30d7ff631f --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import java.io.IOException; +import java.net.InetAddress; +import java.nio.file.Path; +import java.security.SecureRandom; +import java.time.Clock; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import org.apache.hertzbeat.common.runtime.BusinessRuntimeGate; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigDeploymentDetector; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.SetupInstallationPaths; +import org.apache.hertzbeat.manager.setup.identity.DatabaseAccountRepository; +import org.apache.hertzbeat.manager.setup.identity.IdentityInitializationService; +import org.apache.hertzbeat.manager.setup.installation.InstallationCompletionService; +import org.apache.hertzbeat.manager.setup.installation.InstallationRecordRepository; +import org.apache.hertzbeat.manager.setup.installation.LocalInstallationFingerprintStore; +import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; +import org.apache.hertzbeat.manager.setup.runtime.SetupResponseTransition; +import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransitionScheduler; +import org.apache.hertzbeat.manager.setup.security.RemoteSetupUnlock; +import org.apache.hertzbeat.manager.setup.security.SetupHttpUnlockService; +import org.apache.hertzbeat.manager.setup.unattended.SetupPasswordFileLoader; +import org.apache.hertzbeat.manager.setup.unattended.UnattendedSetupInitializer; +import org.apache.hertzbeat.manager.setup.workflow.DefaultSetupWorkflow; +import org.apache.hertzbeat.manager.setup.workflow.HeadlessSetupCoordinator; +import org.apache.hertzbeat.manager.setup.workflow.SetupCompletionCoordinator; +import org.apache.hertzbeat.manager.setup.workflow.SetupConfigurationCoordinator; +import org.apache.hertzbeat.manager.setup.workflow.SetupExportRenderer; +import org.apache.hertzbeat.manager.setup.workflow.SetupMutationSerializer; +import org.apache.hertzbeat.manager.setup.workflow.SetupOperationRegistry; +import org.apache.hertzbeat.manager.setup.workflow.SetupOptionsCoordinator; +import org.apache.hertzbeat.manager.setup.workflow.HeadlessSetupWorkflow; +import org.apache.hertzbeat.manager.setup.workflow.SetupRequestValidator; +import org.apache.hertzbeat.manager.setup.workflow.SetupRuntimeState; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.boot.ApplicationRunner; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.core.env.Environment; + +/** Minimal setup assembly shared by setup-only and full setup-gated contexts. */ +@Configuration(proxyBeanMethods = false) +@Import({SetupController.class, SetupExceptionHandler.class}) +public class SetupApiConfiguration { + @Bean + public ManagedConfigCapability setupManagedConfigCapability(Environment environment) { + return new ManagedConfigDeploymentDetector(SetupInstallationPaths.root(environment)).detect(); + } + + @Bean + public SetupOperationRegistry setupOperationRegistry() { + return new SetupOperationRegistry(Clock.systemUTC()); + } + + @Bean + public SetupResponseTransition setupResponseTransition() { + return new SetupResponseTransition(); + } + + @Bean(destroyMethod = "close") + public SetupRuntimeTransitionScheduler setupRuntimeTransitionScheduler( + SetupRuntimeTransition transition) { + ExecutorService executor = Executors.newSingleThreadExecutor( + Thread.ofPlatform().name("setup-runtime-transition").factory()); + return new SetupRuntimeTransitionScheduler(transition, executor); + } + + @Bean + public SetupRequestValidator setupRequestValidator() { + return new SetupRequestValidator(Clock.systemUTC()); + } + + @Bean + public SetupExportRenderer setupExportRenderer() { + return new SetupExportRenderer(); + } + + @Bean + public SetupConfigurationCoordinator setupConfigurationCoordinator( + Environment environment, SetupOperationRegistry operations) { + return new SetupConfigurationCoordinator( + new ManagedConfigurationTransaction(SetupInstallationPaths.root(environment)), operations); + } + + @Bean + public SetupRuntimeState setupRuntimeState(Environment environment, BusinessRuntimeGate gate, + ManagedConfigCapability capability, + ObjectProvider accountProvider, + ObjectProvider installationProvider) { + Path root = SetupInstallationPaths.root(environment); + return new SetupRuntimeStateFactory().create(environment, root, + bindAddress(environment.getProperty("server.address")), gate, capability, + accountProvider.stream().findFirst(), installationProvider.stream().findFirst()); + } + + @Bean(destroyMethod = "close") + public SetupHttpUnlockService setupHttpUnlockService( + Environment environment, SetupRuntimeState state) throws IOException { + Path codeFile = SetupInstallationPaths.root(environment).resolve("data/config/setup-unlock-code"); + InetAddress bindAddress = bindAddress(environment.getProperty("server.address")); + Clock clock = Clock.systemUTC(); + return new SetupHttpUnlockService(new RemoteSetupUnlock(codeFile, clock, new SecureRandom()), + bindAddress, state, clock); + } + + @Bean + public SetupMutationSerializer setupMutationSerializer() { + return new SetupMutationSerializer(); + } + + @Bean + public DefaultSetupWorkflow setupWorkflow(Environment environment, SetupRuntimeState state, + SetupRequestValidator validator, SetupConfigurationCoordinator configuration, + SetupOperationRegistry operations, ManagedConfigCapability capability, + ObjectProvider identityProvider, + ObjectProvider installationProvider, + SetupMutationSerializer mutations) { + Optional completion = completion( + environment, installationProvider.stream().findFirst()); + return new DefaultSetupWorkflow(state, validator, configuration, operations, capability, + identityProvider.stream().findFirst(), completion, + new SetupOptionsCoordinator(new ManagedConfigurationTransaction( + SetupInstallationPaths.root(environment))), Clock.systemUTC(), mutations); + } + + @Bean + public HeadlessSetupWorkflow headlessSetupWorkflow( + Environment environment, SetupRuntimeState state, SetupRequestValidator validator, + SetupConfigurationCoordinator configuration, ManagedConfigCapability capability, + ObjectProvider identityProvider, + ObjectProvider installationProvider, + SetupMutationSerializer mutations) { + return new HeadlessSetupCoordinator(state, validator, configuration, capability, + identityProvider.stream().findFirst(), + completion(environment, installationProvider.stream().findFirst()), mutations); + } + + @Bean + public ApplicationRunner unattendedSetupRunner( + HeadlessSetupWorkflow workflow, Environment environment, SetupRuntimeTransitionScheduler scheduler) { + UnattendedSetupInitializer initializer = new UnattendedSetupInitializer( + workflow, environment, new SetupPasswordFileLoader(), Optional.of(scheduler)); + return arguments -> initializer.initialize(); + } + + @Bean + public ApplicationRunner completedInstallationConvergenceRunner( + BusinessRuntimeGate gate, SetupRuntimeState state, SetupRuntimeTransitionScheduler scheduler) { + return arguments -> { + if (gate.mode() == RuntimeMode.FULL_SETUP_GATED && state.phase() == SetupPhase.COMPLETE) { + scheduler.installationCompleted(); + } + }; + } + + private Optional completion( + Environment environment, Optional installations) { + Path fingerprint = SetupInstallationPaths.root(environment) + .resolve("data/config/.installation-fingerprint"); + return installations.map(service -> new SetupCompletionCoordinator( + new LocalInstallationFingerprintStore(fingerprint, new SecureRandom()), service)); + } + + static InetAddress bindAddress(String configured) { + try { + return configured == null || configured.isBlank() + ? InetAddress.getByName("0.0.0.0") : InetAddress.getByName(configured); + } catch (IOException failure) { + throw new IllegalStateException("Setup bind address is invalid"); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java index decdd2c8f0..824c4a75a9 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java @@ -273,7 +273,8 @@ public final class SetupApiContract { EXTERNAL_APPLY_REQUIRED("external_apply_required"), RESTART_REQUIRED("restart_required"), PUBLIC_ADDRESS_PLAINTEXT("public_address_plaintext"), - MAIL_SECURITY_NONE("mail_security_none"); + MAIL_SECURITY_NONE("mail_security_none"), + H2_NON_PRODUCTION("h2_non_production"); private final String value; @@ -299,7 +300,16 @@ public final class SetupApiContract { @NotNull @Valid ManagementDatabaseSummary managementDatabase, @NotNull @Valid TelemetryStoreSummary telemetryStore, boolean administratorConfigured, - @NotNull @Valid OptionalConfigurationSummary optional) { + @NotNull @Valid OptionalConfigurationSummary optional, + @NotNull List pendingWarnings) { + + public StatusResponse(SetupPhase phase, Instant observedAt, SetupAccess access, ApplyMode applyMode, + boolean writableManagedConfig, String operationId, SetupErrorCode errorCode, + ManagementDatabaseSummary managementDatabase, TelemetryStoreSummary telemetryStore, + boolean administratorConfigured, OptionalConfigurationSummary optional) { + this(phase, observedAt, access, applyMode, writableManagedConfig, operationId, errorCode, + managementDatabase, telemetryStore, administratorConfigured, optional, List.of()); + } } /** Secret-free metadata database summary. */ diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiException.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiException.java new file mode 100644 index 0000000000..50ffaf51cc --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiException.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.springframework.http.HttpStatus; + +/** Typed internal setup failure; its message is deliberately just the stable wire code. */ +public final class SetupApiException extends IllegalStateException { + private final SetupErrorCode errorCode; + private final HttpStatus status; + + public SetupApiException(SetupErrorCode errorCode, HttpStatus status) { + super(errorCode.value()); + this.errorCode = errorCode; + this.status = status; + } + + public SetupErrorCode errorCode() { + return errorCode; + } + + public HttpStatus status() { + return status; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupController.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupController.java new file mode 100644 index 0000000000..44e82e9c11 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupController.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import jakarta.servlet.http.HttpServletRequest; +import jakarta.validation.Valid; +import java.io.IOException; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.AdministratorRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.AdministratorResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.CompleteRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.CompleteResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OperationResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.StatusResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.UnlockRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.UnlockResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidateRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationResponse; +import org.apache.hertzbeat.manager.setup.runtime.SetupResponseTransition; +import org.apache.hertzbeat.manager.setup.security.SetupHttpUnlockService; +import org.apache.hertzbeat.manager.setup.workflow.SetupExportRenderer; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; + +/** Transport-only adapter for the frozen first-install setup routes. */ +@RestController +public class SetupController { + private final SetupWorkflow workflow; + private final SetupHttpUnlockService unlockService; + private final SetupResponseTransition responseTransition; + private final SetupExportRenderer exportRenderer; + + public SetupController(SetupWorkflow workflow, SetupHttpUnlockService unlockService, + SetupResponseTransition responseTransition, SetupExportRenderer exportRenderer) { + this.workflow = workflow; + this.unlockService = unlockService; + this.responseTransition = responseTransition; + this.exportRenderer = exportRenderer; + } + + @GetMapping(SetupApiContract.STATUS_PATH) + public ResponseEntity status() { + return SetupHttpContract.noStore().body(workflow.status()); + } + + @PostMapping(SetupApiContract.UNLOCK_PATH) + public ResponseEntity unlock( + @Valid @RequestBody UnlockRequest request, HttpServletRequest servletRequest) throws IOException { + if (unlockService.requiresUnlock(servletRequest)) { + var exchange = unlockService.redeem(request, servletRequest); + return SetupHttpContract.noStore().header(HttpHeaders.SET_COOKIE, + exchange.cookie().toString()).body(exchange.response()); + } + return SetupHttpContract.noStore().body(workflow.unlock(request)); + } + + @PostMapping(SetupApiContract.VALIDATE_PATH) + public ResponseEntity validate(@Valid @RequestBody ValidateRequest request) { + return SetupHttpContract.noStore().body(workflow.validate(request)); + } + + @PostMapping(SetupApiContract.CONFIGURATION_PATH) + public ResponseEntity configure( + @Valid @RequestBody ConfigurationRequest request, HttpServletRequest servletRequest) { + ConfigurationResponse response = workflow.configure(request); + if (response.phase() == SetupPhase.APPLICATION_STARTING) { + responseTransition.arm(servletRequest); + } + return SetupHttpContract.noStore().body(response); + } + + @GetMapping(SetupApiContract.OPERATION_PATH) + public ResponseEntity operation(@PathVariable String operationId) { + OperationResponse response = workflow.operation(operationId); + if (response == null) { + throw new SetupApiException(SetupErrorCode.OPERATION_NOT_FOUND, HttpStatus.NOT_FOUND); + } + return SetupHttpContract.noStore().body(response); + } + + @PostMapping(SetupApiContract.ADMINISTRATOR_PATH) + public ResponseEntity administrator(@Valid @RequestBody AdministratorRequest request) { + return SetupHttpContract.noStore().body(workflow.createAdministrator(request)); + } + + @PostMapping(SetupApiContract.OPTIONS_PATH) + public ResponseEntity options(@Valid @RequestBody OptionsRequest request) { + return SetupHttpContract.noStore().body(workflow.configureOptions(request)); + } + + @PostMapping(SetupApiContract.EXPORT_PATH) + public ResponseEntity export(@Valid @RequestBody ExportRequest request) { + var metadata = workflow.prepareExport(request); + var artifact = exportRenderer.render(request, metadata); + StreamingResponseBody body = output -> artifact.content().writeTo(output); + return SetupHttpContract.noStore() + .header(HttpHeaders.CONTENT_DISPOSITION, + "attachment; filename=\"" + artifact.fileName() + "\"") + .header(HttpHeaders.CONTENT_TYPE, artifact.mediaType()).body(body); + } + + @PostMapping(SetupApiContract.COMPLETE_PATH) + public ResponseEntity complete( + @Valid @RequestBody CompleteRequest request, HttpServletRequest servletRequest) { + CompleteResponse response = workflow.complete(request); + responseTransition.armCompletion(servletRequest); + return SetupHttpContract.noStore().body(response); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupErrorResponse.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupErrorResponse.java new file mode 100644 index 0000000000..c7b1a7c004 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupErrorResponse.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import java.time.Instant; +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Stable setup failure envelope that never carries exception or connection details. */ +public record SetupErrorResponse(SetupErrorCode errorCode, Instant observedAt) { + public SetupErrorResponse { + Objects.requireNonNull(errorCode, "errorCode"); + Objects.requireNonNull(observedAt, "observedAt"); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandler.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandler.java new file mode 100644 index 0000000000..279852a4cd --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandler.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import jakarta.servlet.http.HttpServletRequest; +import java.time.Clock; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.security.SetupUnlockRejected; +import org.apache.hertzbeat.manager.setup.workflow.SetupWorkflowConflict; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.http.converter.HttpMessageNotReadableException; +import org.springframework.web.bind.MethodArgumentNotValidException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; + +/** Owns safe HTTP classification for setup failures. */ +@RestControllerAdvice(assignableTypes = SetupController.class) +public class SetupExceptionHandler { + private final Clock clock; + + public SetupExceptionHandler() { + this(Clock.systemUTC()); + } + + SetupExceptionHandler(Clock clock) { + this.clock = clock; + } + + @ExceptionHandler(SetupApiException.class) + public ResponseEntity apiFailure(SetupApiException failure) { + return response(failure.status(), failure.errorCode()); + } + + @ExceptionHandler(SetupWorkflowConflict.class) + public ResponseEntity workflowConflict(SetupWorkflowConflict ignored) { + return response(HttpStatus.CONFLICT, SetupErrorCode.OPERATION_CONFLICT); + } + + @ExceptionHandler(SetupUnlockRejected.class) + public ResponseEntity unlockRejected(SetupUnlockRejected failure) { + return switch (failure.reason()) { + case INVALID -> response(HttpStatus.FORBIDDEN, SetupErrorCode.SETUP_CODE_INVALID); + case EXPIRED -> response(HttpStatus.FORBIDDEN, SetupErrorCode.SETUP_CODE_EXPIRED); + case RATE_LIMITED -> response(HttpStatus.TOO_MANY_REQUESTS, SetupErrorCode.SETUP_RATE_LIMITED); + }; + } + + @ExceptionHandler({MethodArgumentNotValidException.class, HttpMessageNotReadableException.class}) + public ResponseEntity invalidRequest(Exception ignored, HttpServletRequest request) { + SetupErrorCode code = SetupApiContract.UNLOCK_PATH.equals(request.getRequestURI()) + ? SetupErrorCode.SETUP_CODE_INVALID : SetupErrorCode.OPERATION_CONFLICT; + return response(HttpStatus.BAD_REQUEST, code); + } + + @ExceptionHandler(Exception.class) + public ResponseEntity unexpectedFailure(Exception ignored) { + return response(HttpStatus.INTERNAL_SERVER_ERROR, SetupErrorCode.CONFIG_WRITE_FAILED); + } + + private ResponseEntity response(HttpStatus status, SetupErrorCode code) { + return ResponseEntity.status(status).header("Cache-Control", "no-store") + .body(new SetupErrorResponse(code, clock.instant())); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupRuntimeStateFactory.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupRuntimeStateFactory.java new file mode 100644 index 0000000000..e765bca37a --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupRuntimeStateFactory.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import java.net.InetAddress; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Optional; +import org.apache.hertzbeat.common.runtime.BusinessRuntimeGate; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupAccess; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector; +import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector.State; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; +import org.apache.hertzbeat.manager.setup.identity.DatabaseAccountRepository; +import org.apache.hertzbeat.manager.setup.installation.InstallationConvergenceService; +import org.apache.hertzbeat.manager.setup.installation.InstallationMode; +import org.apache.hertzbeat.manager.setup.installation.InstallationRecordRepository; +import org.apache.hertzbeat.manager.setup.workflow.SetupRuntimeState; +import org.springframework.core.env.Environment; + +/** Derives the initial HTTP setup state from runtime, managed files, identity, and installation truth. */ +final class SetupRuntimeStateFactory { + SetupRuntimeState create(Environment environment, Path root, InetAddress bindAddress, BusinessRuntimeGate gate, + ManagedConfigCapability capability, Optional accounts, + Optional installations) { + var inspection = new ManagedActiveConfigurationInspector(root).inspect(); + var administrator = accounts.flatMap(DatabaseAccountRepository::findByBootstrapSlotIsNotNull); + Optional installationMode = installations.map(repository -> + new InstallationConvergenceService(repository, + root.resolve("data/config/.installation-fingerprint")).classify()); + SetupPhase phase = phase(gate.mode(), inspection.state(), administrator.isPresent(), installationMode); + return new SetupRuntimeState(Clock.systemUTC(), capability, phase, + bindAddress.isLoopbackAddress() ? SetupAccess.LOCAL : SetupAccess.LOCKED, + administrator.isPresent(), administrator.map(account -> account.username()).orElse(null), + new SetupStatusProjectionFactory().create(environment, inspection)); + } + + private static SetupPhase phase(RuntimeMode mode, State inspection, boolean administratorConfigured, + Optional installationMode) { + if (mode == RuntimeMode.RECOVERY || inspection == State.RECOVERY_REQUIRED) { + return SetupPhase.RECOVERY_REQUIRED; + } + return switch (mode) { + case SETUP_ONLY -> inspection == State.LOADABLE + ? SetupPhase.APPLICATION_STARTING : SetupPhase.CONFIGURATION_REQUIRED; + case FULL_SETUP_GATED -> fullSetupPhase(administratorConfigured, installationMode); + case NORMAL -> SetupPhase.COMPLETE; + case RECOVERY -> SetupPhase.RECOVERY_REQUIRED; + }; + } + + private static SetupPhase fullSetupPhase(boolean administratorConfigured, + Optional installationMode) { + if (installationMode.filter(mode -> mode == InstallationMode.FULL).isPresent()) { + return SetupPhase.COMPLETE; + } + if (installationMode.filter(mode -> mode == InstallationMode.RECOVERY).isPresent()) { + return SetupPhase.RECOVERY_REQUIRED; + } + return administratorConfigured + ? SetupPhase.OPTIONAL_CONFIGURATION : SetupPhase.ADMINISTRATOR_REQUIRED; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupStatusProjectionFactory.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupStatusProjectionFactory.java new file mode 100644 index 0000000000..0c1fb2e75b --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupStatusProjectionFactory.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.DATABASE_KIND; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.GREPTIME_ENABLED; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.MAIL_HOST; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.MAIL_SECURITY; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.PUBLIC_BASE_URL; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.RETENTION_LOGS; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.RETENTION_METRICS; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.RETENTION_TRACES; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.SERVER_OTLP_GRPC; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.SERVER_OTLP_HTTP; + +import java.util.Locale; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigSource; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ManagementDatabaseSummary; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionalConfigurationSummary; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreSummary; +import org.apache.hertzbeat.manager.setup.config.EffectiveConfigurationResolver; +import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector.Inspection; +import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector.State; +import org.apache.hertzbeat.manager.setup.config.RestartRequirement; +import org.apache.hertzbeat.manager.setup.workflow.SetupConfigurationProjection; +import org.apache.hertzbeat.manager.setup.workflow.SetupWarningPolicy; +import org.springframework.core.env.Environment; + +/** Builds the secret-free status view from verified managed data and effective Spring precedence. */ +final class SetupStatusProjectionFactory { + private final EffectiveConfigurationResolver resolver = new EffectiveConfigurationResolver(); + + SetupConfigurationProjection create(Environment environment, Inspection inspection) { + if (!environment.containsProperty(DATABASE_KIND)) { + return SetupConfigurationProjection.defaults(); + } + var database = resolver.resolve(environment, DATABASE_KIND, RestartRequirement.RESTART_REQUIRED); + ConfigSource telemetrySource = environment.containsProperty(GREPTIME_ENABLED) + ? resolver.resolve(environment, GREPTIME_ENABLED, RestartRequirement.RESTART_REQUIRED).source() + : database.source(); + boolean managedPresent = inspection.state() == State.LOADABLE; + MetadataDatabaseKind kind = MetadataDatabaseKind.valueOf(database.value().toUpperCase(Locale.ROOT)); + OptionalConfigurationSummary optional = new OptionalConfigurationSummary( + hasText(environment, PUBLIC_BASE_URL), hasText(environment, SERVER_OTLP_HTTP), + hasText(environment, SERVER_OTLP_GRPC), hasRetention(environment), + hasText(environment, MAIL_HOST)); + String mailSecurityValue = environment.getProperty(MAIL_SECURITY); + MailSecurity mailSecurity = mailSecurityValue != null + && MailSecurity.NONE.name().equalsIgnoreCase(mailSecurityValue) ? MailSecurity.NONE : null; + return new SetupConfigurationProjection( + new ManagementDatabaseSummary(kind, managedPresent || database.source() != ConfigSource.BUILT_IN_DEFAULT, + database.source(), false), + new TelemetryStoreSummary(TelemetryStoreKind.GREPTIME, + managedPresent || telemetrySource != ConfigSource.BUILT_IN_DEFAULT, + telemetrySource, false), optional, SetupWarningPolicy.INSTANCE.evaluate( + kind, environment.getProperty(PUBLIC_BASE_URL), mailSecurity)); + } + + private static boolean hasText(Environment environment, String key) { + String value = environment.getProperty(key); + return value != null && !value.isBlank(); + } + + private static boolean hasRetention(Environment environment) { + return environment.containsProperty(RETENTION_METRICS) + || environment.containsProperty(RETENTION_LOGS) + || environment.containsProperty(RETENTION_TRACES); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ApplicationConfigDocumentCodec.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ApplicationConfigDocumentCodec.java index 6b2028d4f1..4d48b0ec77 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ApplicationConfigDocumentCodec.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ApplicationConfigDocumentCodec.java @@ -17,9 +17,28 @@ package org.apache.hertzbeat.manager.setup.config; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.DATABASE_KIND; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.DATASOURCE_URL; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.DATASOURCE_USERNAME; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.GREPTIME_DATABASE; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.GREPTIME_ENABLED; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.GREPTIME_GRPC; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.GREPTIME_HTTP; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.GREPTIME_USERNAME; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.MAIL_HOST; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.MAIL_SECURITY; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.PUBLIC_BASE_URL; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.RETENTION_LOGS; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.RETENTION_METRICS; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.RETENTION_TRACES; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.SERVER_OTLP_GRPC; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.SERVER_OTLP_HTTP; + import java.util.LinkedHashMap; import java.util.Map; +import java.util.Optional; import java.util.Set; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; import org.yaml.snakeyaml.LoaderOptions; import org.yaml.snakeyaml.Yaml; @@ -29,19 +48,20 @@ import org.yaml.snakeyaml.error.YAMLException; final class ApplicationConfigDocumentCodec implements ManagedDocumentCodec { // This exact flat-key allowlist is the boundary that prevents setup from becoming an arbitrary YAML editor. - private static final String DATASOURCE_URL = "spring.datasource.url"; - private static final String DATASOURCE_USERNAME = "spring.datasource.username"; - private static final String DATABASE_KIND = "spring.jpa.database"; private static final String DUCKDB_ENABLED = "warehouse.store.duckdb.enabled"; - private static final String GREPTIME_ENABLED = "warehouse.store.greptime.enabled"; - private static final String GREPTIME_GRPC = "warehouse.store.greptime.grpc-endpoints"; - private static final String GREPTIME_HTTP = "warehouse.store.greptime.http-endpoint"; - private static final String GREPTIME_DATABASE = "warehouse.store.greptime.database"; - private static final String GREPTIME_USERNAME = "warehouse.store.greptime.username"; + private static final String MAIL_PORT = "spring.mail.port"; + private static final String MAIL_USERNAME = "spring.mail.username"; + private static final String MAIL_FROM = "hertzbeat.setup.mail.from-address"; private static final Set REQUIRED_KEYS = Set.of( DATASOURCE_URL, DATASOURCE_USERNAME, DATABASE_KIND, DUCKDB_ENABLED, GREPTIME_ENABLED, GREPTIME_GRPC, GREPTIME_HTTP, GREPTIME_DATABASE); + private static final Set ALLOWED_KEYS = Set.of( + DATASOURCE_URL, DATASOURCE_USERNAME, DATABASE_KIND, + DUCKDB_ENABLED, GREPTIME_ENABLED, GREPTIME_GRPC, GREPTIME_HTTP, + GREPTIME_DATABASE, GREPTIME_USERNAME, PUBLIC_BASE_URL, SERVER_OTLP_HTTP, + SERVER_OTLP_GRPC, RETENTION_METRICS, RETENTION_LOGS, RETENTION_TRACES, + MAIL_HOST, MAIL_PORT, MAIL_SECURITY, MAIL_USERNAME, MAIL_FROM); @Override public byte[] encode(ManagedApplicationConfig value, String generation) { @@ -70,6 +90,23 @@ final class ApplicationConfigDocumentCodec implements ManagedDocumentCodec values.put(GREPTIME_USERNAME, username)); + value.optional().publicAccess().ifPresent(publicAccess -> { + publicAccess.publicBaseUrl().ifPresent(item -> values.put(PUBLIC_BASE_URL, item)); + publicAccess.serverOtlpHttpEndpoint().ifPresent(item -> values.put(SERVER_OTLP_HTTP, item)); + publicAccess.serverOtlpGrpcEndpoint().ifPresent(item -> values.put(SERVER_OTLP_GRPC, item)); + }); + value.optional().retention().ifPresent(retention -> { + putInteger(values, RETENTION_METRICS, retention.metricsDays()); + putInteger(values, RETENTION_LOGS, retention.logsDays()); + putInteger(values, RETENTION_TRACES, retention.tracesDays()); + }); + value.optional().mail().ifPresent(mail -> { + values.put(MAIL_HOST, mail.host()); + values.put(MAIL_PORT, Integer.toString(mail.port())); + values.put(MAIL_SECURITY, mail.security().name()); + mail.username().ifPresent(item -> values.put(MAIL_USERNAME, item)); + values.put(MAIL_FROM, mail.fromAddress()); + }); return values; } @@ -90,8 +127,8 @@ final class ApplicationConfigDocumentCodec implements ManagedDocumentCodec REQUIRED_KEYS.size() + 1 - || (values.size() > REQUIRED_KEYS.size() && !values.containsKey(GREPTIME_USERNAME)) + || !ALLOWED_KEYS.containsAll(values.keySet()) + || !completeMailGroup(values) || !usesSupportedTelemetryStorage(values)) { throw DocumentException.corrupt(); } @@ -102,16 +139,64 @@ final class ApplicationConfigDocumentCodec implements ManagedDocumentCodec(decoded, body.generation()); } + private static ManagedOptionalConfiguration optional(Map values) { + boolean publicPresent = containsAny(values, PUBLIC_BASE_URL, SERVER_OTLP_HTTP, SERVER_OTLP_GRPC); + Optional publicAccess = publicPresent + ? Optional.of(new ManagedOptionalConfiguration.PublicAccessSettings( + optionalText(values, PUBLIC_BASE_URL), optionalText(values, SERVER_OTLP_HTTP), + optionalText(values, SERVER_OTLP_GRPC))) : Optional.empty(); + boolean retentionPresent = containsAny(values, RETENTION_METRICS, RETENTION_LOGS, RETENTION_TRACES); + Optional retention = retentionPresent + ? Optional.of(new ManagedOptionalConfiguration.RetentionSettings( + optionalInteger(values, RETENTION_METRICS), optionalInteger(values, RETENTION_LOGS), + optionalInteger(values, RETENTION_TRACES))) : Optional.empty(); + Optional mail = values.containsKey(MAIL_HOST) + ? Optional.of(new ManagedOptionalConfiguration.MailSettings( + text(values, MAIL_HOST), Integer.parseInt(text(values, MAIL_PORT)), + MailSecurity.valueOf(text(values, MAIL_SECURITY)), optionalText(values, MAIL_USERNAME), + text(values, MAIL_FROM))) : Optional.empty(); + return new ManagedOptionalConfiguration(publicAccess, retention, mail); + } + + private static boolean completeMailGroup(Map values) { + boolean any = containsAny(values, MAIL_HOST, MAIL_PORT, MAIL_SECURITY, MAIL_USERNAME, MAIL_FROM); + return !any || values.keySet().containsAll(Set.of(MAIL_HOST, MAIL_PORT, MAIL_SECURITY, MAIL_FROM)); + } + + private static boolean containsAny(Map values, String... keys) { + for (String key : keys) { + if (values.containsKey(key)) { + return true; + } + } + return false; + } + + private static Optional optionalText(Map values, String key) { + return values.containsKey(key) ? Optional.of(text(values, key)) : Optional.empty(); + } + + private static Integer optionalInteger(Map values, String key) { + return values.containsKey(key) ? Integer.valueOf(text(values, key)) : null; + } + + private static void putInteger(Map values, String key, Integer value) { + if (value != null) { + values.put(key, value.toString()); + } + } + /** * Managed setup configuration has one supported telemetry-storage policy. Checking the values as * well as the keys prevents a hand-edited document from silently enabling an unsupported store. diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedSnapshotStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedSnapshotStore.java index f257b66afa..c49c6e78ed 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedSnapshotStore.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedSnapshotStore.java @@ -74,26 +74,31 @@ final class FileManagedSnapshotStore { void promoteCandidate(T expected, String generation) throws IOException { ensureSafePaths(); - byte[] candidateDocument; - ManagedDocumentCodec.Decoded decoded; + ManagedDocumentCodec.Decoded decoded = null; + CandidateRead activeRead = null; try { - candidateDocument = reader.read(candidate); + byte[] candidateDocument = reader.read(candidate); decoded = codec.decode(candidateDocument); - } catch (ManagedDocumentCodec.DocumentException | IOException failure) { + if (!expected.equals(decoded.value()) || !generation.equals(decoded.generation())) { + throw new IOException("Managed configuration candidate does not match the transaction"); + } + activeRead = readActive(); + if (activeRead.state() == CandidateState.VALID) { + publisher.publish(lastKnownGood, codec.encode( + activeRead.value().orElseThrow(), activeRead.generation().orElseThrow()), ownerOnly); + } else if (activeRead.state() != CandidateState.MISSING) { + throw new IOException("Active managed configuration requires recovery"); + } + publisher.publish(active, candidateDocument, ownerOnly); + publisher.remove(candidate); + } catch (ManagedDocumentCodec.DocumentException failure) { throw new IOException("A valid managed configuration candidate is required"); + } finally { + close(decoded == null ? null : decoded.value()); + if (activeRead != null) { + activeRead.value().ifPresent(FileManagedSnapshotStore::close); + } } - if (!expected.equals(decoded.value()) || !generation.equals(decoded.generation())) { - throw new IOException("Managed configuration candidate does not match the transaction"); - } - CandidateRead activeRead = readActive(); - if (activeRead.state() == CandidateState.VALID) { - publisher.publish(lastKnownGood, codec.encode( - activeRead.value().orElseThrow(), activeRead.generation().orElseThrow()), ownerOnly); - } else if (activeRead.state() != CandidateState.MISSING) { - throw new IOException("Active managed configuration requires recovery"); - } - publisher.publish(active, candidateDocument, ownerOnly); - publisher.remove(candidate); } void restoreActive(Optional previous, Optional generation) throws IOException { @@ -143,4 +148,14 @@ final class FileManagedSnapshotStore { private static boolean isUnsafePath(Path path) { return Files.isSymbolicLink(path); } + + private static void close(Object value) { + if (value instanceof AutoCloseable closeable) { + try { + closeable.close(); + } catch (Exception ignored) { + // Secret cleanup is best-effort and must not mask the persistence outcome. + } + } + } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedActiveConfigurationInspector.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedActiveConfigurationInspector.java index 62c1c7523d..73a5bbefb4 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedActiveConfigurationInspector.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedActiveConfigurationInspector.java @@ -44,15 +44,15 @@ public final class ManagedActiveConfigurationInspector { public Inspection inspect() { CandidateRead application = applicationStore.readActive(); CandidateRead secrets = secretStore.readActive(); - if (application.state() == CandidateState.MISSING && secrets.state() == CandidateState.MISSING) { - return Inspection.absent(); - } - if (application.state() != CandidateState.VALID - || secrets.state() != CandidateState.VALID - || !application.generation().equals(secrets.generation())) { - return Inspection.recoveryRequired(); - } try { + if (application.state() == CandidateState.MISSING && secrets.state() == CandidateState.MISSING) { + return Inspection.absent(); + } + if (application.state() != CandidateState.VALID + || secrets.state() != CandidateState.VALID + || !application.generation().equals(secrets.generation())) { + return Inspection.recoveryRequired(); + } ManagedConfigurationBundle bundle = new ManagedConfigurationBundle( application.value().orElseThrow(), secrets.value().orElseThrow()); return Inspection.loadable( @@ -60,6 +60,8 @@ public final class ManagedActiveConfigurationInspector { SecretConfigDocumentCodec.springProperties(bundle.secrets())); } catch (IllegalArgumentException failure) { return Inspection.recoveryRequired(); + } finally { + ManagedConfigurationTransaction.close(secrets); } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedApplicationConfig.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedApplicationConfig.java index 12cc1ccda6..34385ef43a 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedApplicationConfig.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedApplicationConfig.java @@ -22,11 +22,18 @@ import java.util.Objects; /** Supported non-secret application overlay owned by setup. */ public record ManagedApplicationConfig( MetadataDatabaseSettings metadataDatabase, - GreptimeSettings telemetryStore) { + GreptimeSettings telemetryStore, + ManagedOptionalConfiguration optional) { + + public ManagedApplicationConfig(MetadataDatabaseSettings metadataDatabase, + GreptimeSettings telemetryStore) { + this(metadataDatabase, telemetryStore, ManagedOptionalConfiguration.empty()); + } public ManagedApplicationConfig { Objects.requireNonNull(metadataDatabase, "metadataDatabase"); Objects.requireNonNull(telemetryStore, "telemetryStore"); + Objects.requireNonNull(optional, "optional"); } @Override diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationBundle.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationBundle.java index 7a5241cea5..8ffb244eda 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationBundle.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationBundle.java @@ -20,7 +20,8 @@ package org.apache.hertzbeat.manager.setup.config; import java.util.Objects; /** The only valid unit for applying or loading the separate application and secret documents. */ -public record ManagedConfigurationBundle(ManagedApplicationConfig application, ManagedSecrets secrets) { +public record ManagedConfigurationBundle( + ManagedApplicationConfig application, ManagedSecrets secrets) implements AutoCloseable { public ManagedConfigurationBundle { Objects.requireNonNull(application, "application"); @@ -30,10 +31,20 @@ public record ManagedConfigurationBundle(ManagedApplicationConfig application, M if (telemetryUsername != telemetryPassword) { throw new IllegalArgumentException("Telemetry username and password must be configured together"); } + boolean mailUsername = application.optional().mail() + .flatMap(ManagedOptionalConfiguration.MailSettings::username).isPresent(); + if (mailUsername != secrets.mailPassword().isPresent()) { + throw new IllegalArgumentException("Mail username and password must be configured together"); + } } @Override public String toString() { return "ManagedConfigurationBundle[configured=true, secrets=]"; } + + @Override + public void close() { + secrets.close(); + } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationKeys.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationKeys.java new file mode 100644 index 0000000000..bb43ac2403 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationKeys.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +/** Property names shared by managed document, export, and status boundaries. */ +public final class ManagedConfigurationKeys { + public static final String DATASOURCE_URL = "spring.datasource.url"; + public static final String DATASOURCE_USERNAME = "spring.datasource.username"; + public static final String DATASOURCE_PASSWORD = "spring.datasource.password"; + public static final String DATABASE_KIND = "spring.jpa.database"; + public static final String GREPTIME_ENABLED = "warehouse.store.greptime.enabled"; + public static final String GREPTIME_GRPC = "warehouse.store.greptime.grpc-endpoints"; + public static final String GREPTIME_HTTP = "warehouse.store.greptime.http-endpoint"; + public static final String GREPTIME_DATABASE = "warehouse.store.greptime.database"; + public static final String GREPTIME_USERNAME = "warehouse.store.greptime.username"; + public static final String GREPTIME_PASSWORD = "warehouse.store.greptime.password"; + public static final String PUBLIC_BASE_URL = "hertzbeat.setup.public-base-url"; + public static final String SERVER_OTLP_HTTP = "hertzbeat.setup.server-otlp-http-endpoint"; + public static final String SERVER_OTLP_GRPC = "hertzbeat.setup.server-otlp-grpc-endpoint"; + public static final String RETENTION_METRICS = "hertzbeat.setup.retention.metrics-days"; + public static final String RETENTION_LOGS = "hertzbeat.setup.retention.logs-days"; + public static final String RETENTION_TRACES = "hertzbeat.setup.retention.traces-days"; + public static final String MAIL_HOST = "spring.mail.host"; + public static final String MAIL_SECURITY = "hertzbeat.setup.mail.security"; + + private ManagedConfigurationKeys() { + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationRecovery.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationRecovery.java new file mode 100644 index 0000000000..4895f4e45d --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationRecovery.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.io.IOException; + +/** Resolves explicit interrupted-publication shapes without inferring generation order. */ +final class ManagedConfigurationRecovery { + private final ManagedApplicationConfigStore applicationStore; + private final ManagedSecretStore secretStore; + + ManagedConfigurationRecovery(ManagedApplicationConfigStore applicationStore, + ManagedSecretStore secretStore) { + this.applicationStore = applicationStore; + this.secretStore = secretStore; + } + + ManagedConfigurationTransaction.Outcome recover() { + Snapshots applications = new Snapshots<>( + applicationStore.readActive(), applicationStore.readCandidate(), + applicationStore.readLastKnownGood()); + Snapshots secrets = new Snapshots<>( + secretStore.readActive(), secretStore.readCandidate(), secretStore.readLastKnownGood()); + try { + if (ManagedConfigurationTransaction.formsPair(applications.active(), secrets.active())) { + boolean interrupted = applications.candidate().state() != CandidateState.MISSING + || secrets.candidate().state() != CandidateState.MISSING; + return discardCandidates() + ? (interrupted ? ManagedConfigurationTransaction.Outcome.ROLLED_BACK + : ManagedConfigurationTransaction.Outcome.APPLIED) + : ManagedConfigurationTransaction.Outcome.RECOVERY_REQUIRED; + } + return recoverExplicitPair(applications, secrets); + } finally { + ManagedConfigurationTransaction.close(secrets.active()); + ManagedConfigurationTransaction.close(secrets.candidate()); + ManagedConfigurationTransaction.close(secrets.lastKnownGood()); + } + } + + ManagedConfigurationTransaction.Outcome rollback( + CandidateRead application, + CandidateRead secrets) { + if (!restoreApplication(application) || !restoreSecrets(secrets)) { + return ManagedConfigurationTransaction.Outcome.RECOVERY_REQUIRED; + } + return discardCandidates() ? ManagedConfigurationTransaction.Outcome.ROLLED_BACK + : ManagedConfigurationTransaction.Outcome.RECOVERY_REQUIRED; + } + + boolean discardCandidates() { + boolean discarded = true; + try { + applicationStore.discardCandidate(); + } catch (IOException failure) { + discarded = false; + } + try { + secretStore.discardCandidate(); + } catch (IOException failure) { + discarded = false; + } + return discarded; + } + + private ManagedConfigurationTransaction.Outcome recoverExplicitPair( + Snapshots applications, Snapshots secrets) { + if (ManagedConfigurationTransaction.validPair(applications.active(), secrets.candidate())) { + return finish(promoteSecrets(secrets.candidate()), ManagedConfigurationTransaction.Outcome.APPLIED); + } + if (ManagedConfigurationTransaction.validPair(applications.candidate(), secrets.active())) { + return finish(promoteApplication(applications.candidate()), ManagedConfigurationTransaction.Outcome.APPLIED); + } + if (ManagedConfigurationTransaction.validPair(applications.lastKnownGood(), secrets.active())) { + return finish(restoreApplication(applications.lastKnownGood()), + ManagedConfigurationTransaction.Outcome.ROLLED_BACK); + } + if (ManagedConfigurationTransaction.validPair(applications.active(), secrets.lastKnownGood())) { + return finish(restoreSecrets(secrets.lastKnownGood()), + ManagedConfigurationTransaction.Outcome.ROLLED_BACK); + } + if (ManagedConfigurationTransaction.validPair(applications.lastKnownGood(), secrets.lastKnownGood())) { + boolean restored = restoreApplication(applications.lastKnownGood()) + && restoreSecrets(secrets.lastKnownGood()); + return finish(restored, ManagedConfigurationTransaction.Outcome.ROLLED_BACK); + } + return ManagedConfigurationTransaction.Outcome.RECOVERY_REQUIRED; + } + + private ManagedConfigurationTransaction.Outcome finish( + boolean recovered, ManagedConfigurationTransaction.Outcome outcome) { + return recovered && discardCandidates() + ? outcome : ManagedConfigurationTransaction.Outcome.RECOVERY_REQUIRED; + } + + private boolean promoteApplication(CandidateRead candidate) { + try { + applicationStore.promoteCandidate(candidate.value().orElseThrow(), candidate.generation().orElseThrow()); + return true; + } catch (IOException failure) { + return false; + } + } + + private boolean promoteSecrets(CandidateRead candidate) { + try { + secretStore.promoteCandidate(candidate.value().orElseThrow(), candidate.generation().orElseThrow()); + return true; + } catch (IOException failure) { + return false; + } + } + + private boolean restoreApplication(CandidateRead candidate) { + try { + applicationStore.restoreActive(candidate); + return true; + } catch (IOException failure) { + return false; + } + } + + private boolean restoreSecrets(CandidateRead candidate) { + try { + secretStore.restoreActive(candidate); + return true; + } catch (IOException failure) { + return false; + } + } + + private record Snapshots(CandidateRead active, CandidateRead candidate, + CandidateRead lastKnownGood) { + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransaction.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransaction.java index 4ec61540ee..6eecd48ec4 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransaction.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransaction.java @@ -25,6 +25,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.Objects; +import java.util.Optional; import java.util.UUID; /** Coordinates the application and secret snapshots as one locked, recoverable operation. */ @@ -35,6 +36,7 @@ public final class ManagedConfigurationTransaction { private final ManagedApplicationConfigStore applicationStore; private final ManagedSecretStore secretStore; private final Path lockFile; + private final ManagedConfigurationRecovery recovery; /** Creates the production file transaction rooted at the HertzBeat installation. */ public ManagedConfigurationTransaction(Path installationRoot) { @@ -48,6 +50,7 @@ public final class ManagedConfigurationTransaction { Path installationRoot) { this.applicationStore = Objects.requireNonNull(applicationStore, "applicationStore"); this.secretStore = Objects.requireNonNull(secretStore, "secretStore"); + this.recovery = new ManagedConfigurationRecovery(applicationStore, secretStore); Path root = Objects.requireNonNull(installationRoot, "installationRoot") .toAbsolutePath().normalize(); this.lockFile = root.resolve("data/config").resolve(LOCK_FILE); @@ -59,164 +62,60 @@ public final class ManagedConfigurationTransaction { return withLock(() -> applyLocked(bundle)); } + /** Rewrites the same two-file aggregate while preserving required settings and secrets. */ + public Outcome applyOptions(ManagedOptionalConfiguration options, + Optional mailPassword) throws IOException { + Objects.requireNonNull(options, "options"); + Objects.requireNonNull(mailPassword, "mailPassword"); + return withLock(() -> new ManagedOptionsUpdate(applicationStore, secretStore) + .apply(options, mailPassword, this::applyLocked)); + } + /** Converges interrupted publication only when an explicit complete generation pair exists. */ public Outcome recover() throws IOException { - return withLock(this::recoverLocked); + return withLock(recovery::recover); } private Outcome applyLocked(ManagedConfigurationBundle bundle) throws IOException { CandidateRead previousApplication = applicationStore.readActive(); CandidateRead previousSecrets = secretStore.readActive(); - if (!formsPair(previousApplication, previousSecrets)) { - return Outcome.RECOVERY_REQUIRED; - } - String generation = UUID.randomUUID().toString(); try { - applicationStore.stageCandidate(bundle.application(), generation); - secretStore.stageCandidate(bundle.secrets(), generation); - } catch (IOException failure) { - discardCandidate(applicationStore, failure); - discardCandidate(secretStore, failure); - throw failure; - } - CandidateRead applicationCandidate = applicationStore.readCandidate(); - CandidateRead secretCandidate = secretStore.readCandidate(); - if (!sameGeneration(applicationCandidate, secretCandidate, generation)) { - return discardCandidates() ? Outcome.NOT_APPLIED : Outcome.RECOVERY_REQUIRED; - } - try { - applicationStore.promoteCandidate(bundle.application(), generation); - secretStore.promoteCandidate(bundle.secrets(), generation); - if (!matchesExpectedActive(bundle, generation)) { - return rollbackAfterPromotionFailure(previousApplication, previousSecrets); + if (!formsPair(previousApplication, previousSecrets)) { + return Outcome.RECOVERY_REQUIRED; } - return Outcome.APPLIED; - } catch (IOException ignored) { - return rollbackAfterPromotionFailure(previousApplication, previousSecrets); + String generation = UUID.randomUUID().toString(); + try { + applicationStore.stageCandidate(bundle.application(), generation); + secretStore.stageCandidate(bundle.secrets(), generation); + } catch (IOException failure) { + discardCandidate(applicationStore, failure); + discardCandidate(secretStore, failure); + throw failure; + } + CandidateRead applicationCandidate = applicationStore.readCandidate(); + CandidateRead secretCandidate = secretStore.readCandidate(); + try { + if (!sameGeneration(applicationCandidate, secretCandidate, generation)) { + return recovery.discardCandidates() ? Outcome.NOT_APPLIED : Outcome.RECOVERY_REQUIRED; + } + } finally { + close(secretCandidate); + } + try { + applicationStore.promoteCandidate(bundle.application(), generation); + secretStore.promoteCandidate(bundle.secrets(), generation); + if (!matchesExpectedActive(bundle, generation)) { + return recovery.rollback(previousApplication, previousSecrets); + } + return Outcome.APPLIED; + } catch (IOException ignored) { + return recovery.rollback(previousApplication, previousSecrets); + } + } finally { + close(previousSecrets); } } - private Outcome rollbackAfterPromotionFailure( - CandidateRead previousApplication, - CandidateRead previousSecrets) { - boolean applicationRestored = restoreApplication(previousApplication); - boolean secretsRestored = restoreSecrets(previousSecrets); - if (!applicationRestored || !secretsRestored) { - return Outcome.RECOVERY_REQUIRED; - } - return discardCandidates() ? Outcome.ROLLED_BACK : Outcome.RECOVERY_REQUIRED; - } - - private Outcome recoverLocked() { - Snapshots applications = new Snapshots<>( - applicationStore.readActive(), applicationStore.readCandidate(), - applicationStore.readLastKnownGood()); - Snapshots secrets = new Snapshots<>( - secretStore.readActive(), secretStore.readCandidate(), secretStore.readLastKnownGood()); - - if (formsPair(applications.active(), secrets.active())) { - boolean interrupted = applications.candidate().state() != CandidateState.MISSING - || secrets.candidate().state() != CandidateState.MISSING; - return discardCandidates() - ? (interrupted ? Outcome.ROLLED_BACK : Outcome.APPLIED) - : Outcome.RECOVERY_REQUIRED; - } - - return recoverExplicitPair(applications, secrets); - } - - private Outcome recoverExplicitPair( - Snapshots applications, Snapshots secrets) { - // Crash-state invariant (no generation ordering is inferred): active+candidate and - // candidate+active are the only split-promotion roll-forwards; LKG participates only - // in a complete same-generation rollback pair. Any other shape remains recovery-required. - if (validPair(applications.active(), secrets.candidate())) { - return finishRecovery(promoteSecrets(secrets.candidate()), Outcome.APPLIED); - } - if (validPair(applications.candidate(), secrets.active())) { - return finishRecovery(promoteApplication(applications.candidate()), Outcome.APPLIED); - } - if (validPair(applications.lastKnownGood(), secrets.active())) { - return finishRecovery( - restoreApplication(applications.lastKnownGood()), Outcome.ROLLED_BACK); - } - if (validPair(applications.active(), secrets.lastKnownGood())) { - return finishRecovery( - restoreSecrets(secrets.lastKnownGood()), Outcome.ROLLED_BACK); - } - if (validPair(applications.lastKnownGood(), secrets.lastKnownGood())) { - return finishRecovery(restoreBoth( - applications.lastKnownGood(), secrets.lastKnownGood()), Outcome.ROLLED_BACK); - } - return Outcome.RECOVERY_REQUIRED; - } - - private Outcome finishRecovery(boolean recovered, Outcome outcome) { - boolean candidatesDiscarded = discardCandidates(); - return recovered && candidatesDiscarded ? outcome : Outcome.RECOVERY_REQUIRED; - } - - private boolean restoreBoth( - CandidateRead application, - CandidateRead secrets) { - boolean applicationRestored = restoreApplication(application); - boolean secretsRestored = restoreSecrets(secrets); - return applicationRestored && secretsRestored; - } - - private boolean promoteApplication(CandidateRead candidate) { - try { - applicationStore.promoteCandidate( - candidate.value().orElseThrow(), candidate.generation().orElseThrow()); - return true; - } catch (IOException failure) { - return false; - } - } - - private boolean promoteSecrets(CandidateRead candidate) { - try { - secretStore.promoteCandidate( - candidate.value().orElseThrow(), candidate.generation().orElseThrow()); - return true; - } catch (IOException failure) { - return false; - } - } - - private boolean restoreApplication(CandidateRead previous) { - try { - applicationStore.restoreActive(previous); - return true; - } catch (IOException failure) { - return false; - } - } - - private boolean restoreSecrets(CandidateRead previous) { - try { - secretStore.restoreActive(previous); - return true; - } catch (IOException failure) { - return false; - } - } - - private boolean discardCandidates() { - boolean discarded = true; - try { - applicationStore.discardCandidate(); - } catch (IOException failure) { - discarded = false; - } - try { - secretStore.discardCandidate(); - } catch (IOException failure) { - discarded = false; - } - return discarded; - } - private Outcome withLock(LockedOperation operation) throws IOException { Path directory = lockFile.getParent(); if (Files.isSymbolicLink(lockFile) || Files.isSymbolicLink(directory) @@ -267,26 +166,30 @@ public final class ManagedConfigurationTransaction { private boolean matchesExpectedActive(ManagedConfigurationBundle bundle, String generation) { CandidateRead application = applicationStore.readActive(); CandidateRead secrets = secretStore.readActive(); - return sameGeneration(application, secrets, generation) - && application.value().filter(bundle.application()::equals).isPresent() - && secrets.value().filter(bundle.secrets()::equals).isPresent(); + try { + return sameGeneration(application, secrets, generation) + && application.value().filter(bundle.application()::equals).isPresent() + && secrets.value().filter(bundle.secrets()::equals).isPresent(); + } finally { + close(secrets); + } } - private static boolean formsPair(CandidateRead left, CandidateRead right) { + static boolean formsPair(CandidateRead left, CandidateRead right) { if (left.state() == CandidateState.MISSING && right.state() == CandidateState.MISSING) { return true; } return validPair(left, right); } - private static boolean validPair(CandidateRead left, CandidateRead right) { + static boolean validPair(CandidateRead left, CandidateRead right) { return left.state() == CandidateState.VALID && right.state() == CandidateState.VALID && left.generation().equals(right.generation()); } - private record Snapshots(CandidateRead active, CandidateRead candidate, - CandidateRead lastKnownGood) { + static void close(CandidateRead secrets) { + secrets.value().ifPresent(ManagedSecrets::close); } @FunctionalInterface @@ -294,6 +197,11 @@ public final class ManagedConfigurationTransaction { Outcome run() throws IOException; } + @FunctionalInterface + interface Publisher { + Outcome publish(ManagedConfigurationBundle bundle) throws IOException; + } + /** Stable outcome without exception details or secret content. */ public enum Outcome { APPLIED, diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionalConfiguration.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionalConfiguration.java new file mode 100644 index 0000000000..9beaf0ddcb --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionalConfiguration.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.util.Objects; +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; + +/** Typed optional setup overlay kept in the existing managed application document. */ +public record ManagedOptionalConfiguration( + Optional publicAccess, + Optional retention, + Optional mail) { + + public ManagedOptionalConfiguration { + Objects.requireNonNull(publicAccess, "publicAccess"); + Objects.requireNonNull(retention, "retention"); + Objects.requireNonNull(mail, "mail"); + } + + public static ManagedOptionalConfiguration empty() { + return new ManagedOptionalConfiguration(Optional.empty(), Optional.empty(), Optional.empty()); + } + + /** Optional externally advertised operator and OTLP endpoints. */ + public record PublicAccessSettings( + Optional publicBaseUrl, + Optional serverOtlpHttpEndpoint, + Optional serverOtlpGrpcEndpoint) { + public PublicAccessSettings { + Objects.requireNonNull(publicBaseUrl, "publicBaseUrl"); + Objects.requireNonNull(serverOtlpHttpEndpoint, "serverOtlpHttpEndpoint"); + Objects.requireNonNull(serverOtlpGrpcEndpoint, "serverOtlpGrpcEndpoint"); + } + } + + /** Optional retention periods by signal family. */ + public record RetentionSettings(Integer metricsDays, Integer logsDays, Integer tracesDays) { + public RetentionSettings { + requirePositive(metricsDays); + requirePositive(logsDays); + requirePositive(tracesDays); + } + + private static void requirePositive(Integer days) { + if (days != null && days <= 0) { + throw new IllegalArgumentException("Retention must be positive"); + } + } + } + + /** Non-secret mail transport settings; its password remains in managed secrets. */ + public record MailSettings(String host, int port, MailSecurity security, + Optional username, String fromAddress) { + public MailSettings { + Objects.requireNonNull(host, "host"); + Objects.requireNonNull(security, "security"); + Objects.requireNonNull(username, "username"); + Objects.requireNonNull(fromAddress, "fromAddress"); + if (port <= 0 || port > 65_535 || username.filter(String::isBlank).isPresent()) { + throw new IllegalArgumentException("Mail settings are invalid"); + } + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionsUpdate.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionsUpdate.java new file mode 100644 index 0000000000..e0076eaa71 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionsUpdate.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Optional; + +/** Rebuilds the managed two-file aggregate for an optional-settings update. */ +final class ManagedOptionsUpdate { + private final ManagedApplicationConfigStore applicationStore; + private final ManagedSecretStore secretStore; + + ManagedOptionsUpdate(ManagedApplicationConfigStore applicationStore, ManagedSecretStore secretStore) { + this.applicationStore = applicationStore; + this.secretStore = secretStore; + } + + ManagedConfigurationTransaction.Outcome apply(ManagedOptionalConfiguration options, + Optional mailPassword, + ManagedConfigurationTransaction.Publisher publisher) + throws IOException { + CandidateRead application = applicationStore.readActive(); + CandidateRead secrets = secretStore.readActive(); + if (!ManagedConfigurationTransaction.validPair(application, secrets)) { + ManagedConfigurationTransaction.close(secrets); + return ManagedConfigurationTransaction.Outcome.RECOVERY_REQUIRED; + } + ManagedApplicationConfig currentApplication = application.value().orElseThrow(); + ManagedSecrets currentSecrets = secrets.value().orElseThrow(); + try { + ManagedSecrets updatedSecrets = copyWithMailPassword(currentSecrets, mailPassword); + try { + return publisher.publish(new ManagedConfigurationBundle( + new ManagedApplicationConfig(currentApplication.metadataDatabase(), + currentApplication.telemetryStore(), options), updatedSecrets)); + } finally { + updatedSecrets.close(); + } + } finally { + currentSecrets.close(); + } + } + + private static ManagedSecrets copyWithMailPassword( + ManagedSecrets current, Optional mailPassword) { + return new ManagedSecrets(copy(current.metadataDatabasePassword()), + current.telemetryPassword().map(ManagedOptionsUpdate::copy), + mailPassword.map(ManagedOptionsUpdate::copy)); + } + + private static SecretValue copy(SecretValue secret) { + char[] clear = secret.copy(); + try { + return SecretValue.of(clear); + } finally { + Arrays.fill(clear, '\0'); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedSecrets.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedSecrets.java index a492f32d2d..c33d2c98f2 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedSecrets.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedSecrets.java @@ -21,11 +21,17 @@ import java.util.Objects; import java.util.Optional; /** Setup-owned secrets stored outside the managed application overlay. */ -public record ManagedSecrets(SecretValue metadataDatabasePassword, Optional telemetryPassword) { +public record ManagedSecrets(SecretValue metadataDatabasePassword, Optional telemetryPassword, + Optional mailPassword) implements AutoCloseable { + + public ManagedSecrets(SecretValue metadataDatabasePassword, Optional telemetryPassword) { + this(metadataDatabasePassword, telemetryPassword, Optional.empty()); + } public ManagedSecrets { Objects.requireNonNull(metadataDatabasePassword, "metadataDatabasePassword"); Objects.requireNonNull(telemetryPassword, "telemetryPassword"); + Objects.requireNonNull(mailPassword, "mailPassword"); } public static ManagedSecrets withoutTelemetryPassword(SecretValue metadataDatabasePassword) { @@ -39,6 +45,14 @@ public record ManagedSecrets(SecretValue metadataDatabasePassword, Optional, telemetryPassword=]"; + return "ManagedSecrets[metadataDatabasePassword=, telemetryPassword=, " + + "mailPassword=]"; + } + + @Override + public void close() { + metadataDatabasePassword.close(); + telemetryPassword.ifPresent(SecretValue::close); + mailPassword.ifPresent(SecretValue::close); } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SecretConfigDocumentCodec.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SecretConfigDocumentCodec.java index efe0328b77..d3c5520588 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SecretConfigDocumentCodec.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SecretConfigDocumentCodec.java @@ -17,31 +17,37 @@ package org.apache.hertzbeat.manager.setup.config; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.DATASOURCE_PASSWORD; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.GREPTIME_PASSWORD; + import java.io.IOException; import java.io.StringReader; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Optional; import java.util.Properties; import java.util.Set; final class SecretConfigDocumentCodec implements ManagedDocumentCodec { - private static final String METADATA_PASSWORD = "spring.datasource.password"; - private static final String TELEMETRY_PASSWORD = "warehouse.store.greptime.password"; + private static final String MAIL_PASSWORD = "spring.mail.password"; @Override public byte[] encode(ManagedSecrets value, String generation) { StringBuilder body = new StringBuilder(); - append(body, METADATA_PASSWORD, value.metadataDatabasePassword()); - value.telemetryPassword().ifPresent(secret -> append(body, TELEMETRY_PASSWORD, secret)); + append(body, DATASOURCE_PASSWORD, value.metadataDatabasePassword()); + value.telemetryPassword().ifPresent(secret -> append(body, GREPTIME_PASSWORD, secret)); + value.mailPassword().ifPresent(secret -> append(body, MAIL_PASSWORD, secret)); return Integrity.envelope(body.toString(), generation); } static Map springProperties(ManagedSecrets value) { Map properties = new LinkedHashMap<>(); - properties.put(METADATA_PASSWORD, springLiteral(value.metadataDatabasePassword())); + properties.put(DATASOURCE_PASSWORD, springLiteral(value.metadataDatabasePassword())); value.telemetryPassword().ifPresent( - secret -> properties.put(TELEMETRY_PASSWORD, springLiteral(secret))); + secret -> properties.put(GREPTIME_PASSWORD, springLiteral(secret))); + value.mailPassword().ifPresent(secret -> properties.put(MAIL_PASSWORD, springLiteral(secret))); return Map.copyOf(properties); } @@ -56,23 +62,32 @@ final class SecretConfigDocumentCodec implements ManagedDocumentCodec telemetry = Optional.empty(); + Optional mail = Optional.empty(); try { - SecretValue metadata = SecretValue.of( - removeSpringPlaceholderEscapes(properties.getProperty(METADATA_PASSWORD))); - decoded = properties.containsKey(TELEMETRY_PASSWORD) - ? ManagedSecrets.withTelemetryPassword( - metadata, SecretValue.of(removeSpringPlaceholderEscapes( - properties.getProperty(TELEMETRY_PASSWORD)))) - : ManagedSecrets.withoutTelemetryPassword(metadata); + metadata = SecretValue.of( + removeSpringPlaceholderEscapes(properties.getProperty(DATASOURCE_PASSWORD))); + telemetry = properties.containsKey(GREPTIME_PASSWORD) + ? Optional.of(SecretValue.of(removeSpringPlaceholderEscapes( + properties.getProperty(GREPTIME_PASSWORD)))) : Optional.empty(); + mail = properties.containsKey(MAIL_PASSWORD) + ? Optional.of(SecretValue.of(removeSpringPlaceholderEscapes( + properties.getProperty(MAIL_PASSWORD)))) : Optional.empty(); + return new Decoded<>(new ManagedSecrets(metadata, telemetry, mail), body.generation()); } catch (IllegalArgumentException exception) { + if (metadata != null) { + metadata.close(); + } + telemetry.ifPresent(SecretValue::close); + mail.ifPresent(SecretValue::close); throw DocumentException.invalid(); } - return new Decoded<>(decoded, body.generation()); } private static void append(StringBuilder body, String key, SecretValue secret) { @@ -80,7 +95,7 @@ final class SecretConfigDocumentCodec implements ManagedDocumentCodec]"; diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SensitiveExportContent.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SensitiveExportContent.java index 49642c202d..bbccd8a7f1 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SensitiveExportContent.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SensitiveExportContent.java @@ -17,8 +17,12 @@ package org.apache.hertzbeat.manager.setup.config; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Arrays; + /** Defensive secret-bearing export bytes that never render their content. */ -public final class SensitiveExportContent { +public final class SensitiveExportContent implements AutoCloseable { private final byte[] content; @@ -37,6 +41,20 @@ public final class SensitiveExportContent { return content.clone(); } + public synchronized void writeTo(OutputStream output) throws IOException { + try { + output.write(content); + output.flush(); + } finally { + close(); + } + } + + @Override + public synchronized void close() { + Arrays.fill(content, (byte) 0); + } + @Override public String toString() { return "SensitiveExportContent[]"; diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SetupInstallationPaths.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SetupInstallationPaths.java new file mode 100644 index 0000000000..d974d6822d --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SetupInstallationPaths.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.nio.file.Path; +import org.springframework.core.env.Environment; + +/** Shared installation-root names used by startup loading and setup writers. */ +public final class SetupInstallationPaths { + public static final String ROOT_PROPERTY = "hertzbeat.internal.installation-root"; + + private SetupInstallationPaths() { + } + + public static Path root(Environment environment) { + return Path.of(environment.getProperty(ROOT_PROPERTY, ".")).toAbsolutePath().normalize(); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/DatabaseAccountRepository.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/DatabaseAccountRepository.java index a103a9ed65..8ceb984342 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/DatabaseAccountRepository.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/DatabaseAccountRepository.java @@ -36,4 +36,6 @@ public interface DatabaseAccountRepository extends JpaRepository findByBootstrapSlotIsNotNull(); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationConvergenceService.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationConvergenceService.java new file mode 100644 index 0000000000..1bfcbb366a --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationConvergenceService.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.installation; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.security.SecureRandom; +import java.util.Optional; + +/** Compares the durable database marker with the owner-only local fingerprint. */ +public final class InstallationConvergenceService { + private final InstallationRecordRepository records; + private final Path fingerprintPath; + + public InstallationConvergenceService(InstallationRecordRepository records, Path fingerprintPath) { + this.records = records; + this.fingerprintPath = fingerprintPath.toAbsolutePath().normalize(); + } + + public InstallationMode classify() { + try { + Optional fingerprint = + new LocalInstallationFingerprintStore(fingerprintPath, new SecureRandom()).read(); + if (fingerprint.isEmpty() && Files.exists(fingerprintPath, LinkOption.NOFOLLOW_LINKS)) { + return InstallationMode.RECOVERY; + } + return new InstallationClassifier().classify( + DatabasePresence.HERTZBEAT_SCHEMA, + records.findById(InstallationRecord.SINGLETON_ID), fingerprint); + } catch (IOException | RuntimeException failure) { + return InstallationMode.RECOVERY; + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupResponseTransition.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupResponseTransition.java new file mode 100644 index 0000000000..296414a097 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupResponseTransition.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.runtime; + +import jakarta.servlet.ServletRequest; + +/** Marks one successful setup response for a transition after serialization and commit. */ +public final class SetupResponseTransition { + private static final String ATTRIBUTE = SetupResponseTransition.class.getName() + ".transition"; + + public void arm(ServletRequest request) { + request.setAttribute(ATTRIBUTE, Transition.CONFIGURATION_APPLIED); + } + + public void armCompletion(ServletRequest request) { + request.setAttribute(ATTRIBUTE, Transition.INSTALLATION_COMPLETED); + } + + Transition consume(ServletRequest request) { + Object transition = request.getAttribute(ATTRIBUTE); + if (!(transition instanceof Transition selected)) { + return null; + } + request.removeAttribute(ATTRIBUTE); + return selected; + } + + enum Transition { CONFIGURATION_APPLIED, INSTALLATION_COMPLETED } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupResponseTransitionFilter.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupResponseTransitionFilter.java new file mode 100644 index 0000000000..55fcd707b8 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupResponseTransitionFilter.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.runtime; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import org.springframework.web.filter.OncePerRequestFilter; + +/** Commits a successful configuration response before scheduling the destructive context transition. */ +public final class SetupResponseTransitionFilter extends OncePerRequestFilter { + private final SetupRuntimeTransitionScheduler scheduler; + private final SetupResponseTransition responseTransition; + + public SetupResponseTransitionFilter( + SetupRuntimeTransitionScheduler scheduler, SetupResponseTransition responseTransition) { + this.scheduler = scheduler; + this.responseTransition = responseTransition; + } + + @Override + protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain chain) + throws ServletException, IOException { + chain.doFilter(request, response); + SetupResponseTransition.Transition transition = responseTransition.consume(request); + if (transition != null) { + response.flushBuffer(); + if (transition == SetupResponseTransition.Transition.INSTALLATION_COMPLETED) { + scheduler.installationCompleted(); + } else { + scheduler.configurationApplied(); + } + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeAccessConfiguration.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeAccessConfiguration.java index 4cd9442a64..5fd232be2f 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeAccessConfiguration.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeAccessConfiguration.java @@ -17,7 +17,10 @@ package org.apache.hertzbeat.manager.setup.runtime; +import java.time.Clock; import org.apache.hertzbeat.common.runtime.BusinessRuntimeGate; +import org.apache.hertzbeat.manager.setup.security.SetupHttpUnlockService; +import org.apache.hertzbeat.manager.setup.security.SetupWriteAccessFilter; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.web.servlet.FilterRegistrationBean; import org.springframework.context.annotation.Bean; @@ -42,4 +45,33 @@ public class SetupRuntimeAccessConfiguration { registration.addUrlPatterns("/*"); return registration; } + + @Bean + public SetupWriteAccessFilter setupWriteAccessFilter(SetupHttpUnlockService unlock) { + return new SetupWriteAccessFilter(unlock, Clock.systemUTC()); + } + + @Bean + public FilterRegistrationBean setupWriteAccessFilterRegistration( + SetupWriteAccessFilter filter) { + FilterRegistrationBean registration = new FilterRegistrationBean<>(filter); + registration.setOrder(Ordered.HIGHEST_PRECEDENCE + 2); + registration.addUrlPatterns("/api/setup/*"); + return registration; + } + + @Bean + public SetupResponseTransitionFilter setupResponseTransitionFilter( + SetupRuntimeTransitionScheduler scheduler, SetupResponseTransition responseTransition) { + return new SetupResponseTransitionFilter(scheduler, responseTransition); + } + + @Bean + public FilterRegistrationBean setupResponseTransitionFilterRegistration( + SetupResponseTransitionFilter filter) { + FilterRegistrationBean registration = new FilterRegistrationBean<>(filter); + registration.setOrder(Ordered.HIGHEST_PRECEDENCE + 3); + registration.addUrlPatterns("/api/setup/configuration", "/api/setup/complete"); + return registration; + } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransition.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransition.java index 61b31645b2..59c0b37458 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransition.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransition.java @@ -26,5 +26,9 @@ package org.apache.hertzbeat.manager.setup.runtime; @FunctionalInterface public interface SetupRuntimeTransition { + default void configurationApplied() { + completeSetup(); + } + void completeSetup(); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransitionScheduler.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransitionScheduler.java new file mode 100644 index 0000000000..0df3073c4f --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransitionScheduler.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.runtime; + +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; + +/** Serializes deferred context transitions and never runs them before application readiness. */ +public final class SetupRuntimeTransitionScheduler implements AutoCloseable { + private final SetupRuntimeTransition transition; + private final Executor executor; + private boolean ready; + private boolean closed; + private Transition running; + private Transition pending; + + public SetupRuntimeTransitionScheduler(SetupRuntimeTransition transition, Executor executor) { + this.transition = transition; + this.executor = executor; + } + + public synchronized void configurationApplied() { + request(Transition.CONFIGURATION); + } + + public synchronized void installationCompleted() { + request(Transition.COMPLETION); + } + + @EventListener + public synchronized void onApplicationReady(ApplicationReadyEvent ignored) { + ready = true; + dispatchIfReady(); + } + + private void dispatchIfReady() { + if (closed || !ready || running != null || pending == null) { + return; + } + Transition selected = pending; + pending = null; + running = selected; + executor.execute(() -> run(selected)); + } + + private void request(Transition requested) { + if (closed + || running == Transition.COMPLETION + || running == requested + || pending == Transition.COMPLETION) { + return; + } + pending = requested; + dispatchIfReady(); + } + + private void run(Transition selected) { + try { + if (selected == Transition.COMPLETION) { + transition.completeSetup(); + } else { + transition.configurationApplied(); + } + } finally { + synchronized (this) { + running = null; + dispatchIfReady(); + } + } + } + + @Override + public synchronized void close() { + closed = true; + pending = null; + if (executor instanceof ExecutorService service) { + service.shutdown(); + } + } + + private enum Transition { CONFIGURATION, COMPLETION } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/RemoteSetupUnlock.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/RemoteSetupUnlock.java index 9e0c51ca1a..16c76c68a6 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/RemoteSetupUnlock.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/RemoteSetupUnlock.java @@ -87,7 +87,17 @@ public final class RemoteSetupUnlock { Arrays.fill(encodedCode, (byte) 0); } } - LOGGER.warn("Remote setup requires the one-time unlock file at {}", codeFile); + LOGGER.warn("Setup unlock proof file created at {}", codeFile); + } + + /** Keeps an active proof or session intact and renews it only after expiry. */ + public synchronized boolean ensureOpen() throws IOException { + Instant now = clock.instant(); + if (expiresAt != null && now.isBefore(expiresAt) && (codeDigest != null || sessionDigest != null)) { + return false; + } + open(); + return true; } private void removeStaleCodeFile() throws IOException { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupHttpUnlockService.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupHttpUnlockService.java new file mode 100644 index 0000000000..067c3cad7c --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupHttpUnlockService.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.security; + +import jakarta.servlet.http.HttpServletRequest; +import java.io.IOException; +import java.net.InetAddress; +import java.time.Clock; +import java.util.Arrays; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupAccess; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.UnlockRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.UnlockResponse; +import org.apache.hertzbeat.manager.setup.workflow.SetupRuntimeState; +import org.springframework.http.ResponseCookie; + +/** Security orchestration for remote setup proof redemption and cookie issuance. */ +public final class SetupHttpUnlockService implements AutoCloseable { + private final RemoteSetupUnlock unlock; + private final InetAddress bindAddress; + private final SetupRuntimeState state; + private final Clock clock; + private final SetupRequestSecurityPolicy requestPolicy; + + public SetupHttpUnlockService(RemoteSetupUnlock unlock, InetAddress bindAddress, + SetupRuntimeState state, Clock clock) throws IOException { + this.unlock = unlock; + this.bindAddress = bindAddress; + this.state = state; + this.clock = clock; + this.requestPolicy = new SetupRequestSecurityPolicy(); + if (state.phase() != SetupPhase.COMPLETE && unlock.requiresUnlock(bindAddress)) { + ensureProof(); + } + } + + public boolean requiresUnlock() { + return state.phase() != SetupPhase.COMPLETE && unlock.requiresUnlock(bindAddress); + } + + public boolean requiresUnlock(HttpServletRequest request) { + var context = requestPolicy.inspect(request); + boolean required = state.phase() != SetupPhase.COMPLETE + && (unlock.requiresUnlock(bindAddress) || context.requiresProofOnLoopback()); + if (required) { + try { + ensureProof(); + } catch (IOException failure) { + throw new IllegalStateException("Setup unlock proof is unavailable", failure); + } + } + return required; + } + + synchronized void ensureProof() throws IOException { + if (unlock.ensureOpen() && unlock.requiresUnlock(bindAddress)) { + state.locked(); + } + } + + public boolean secureCookie(HttpServletRequest request) { + return requestPolicy.secureCookie(request); + } + + public synchronized UnlockExchange redeem( + UnlockRequest request, HttpServletRequest servletRequest) throws IOException { + var context = requestPolicy.inspect(servletRequest); + if (!requiresUnlock(servletRequest)) { + throw new SetupUnlockRejected(SetupUnlockRejected.Reason.INVALID); + } + char[] code = request.code().toCharArray(); + try { + SetupAccessSession session = unlock.redeem(context.remoteAddress(), new SetupUnlockCode(code)); + if (unlock.requiresUnlock(bindAddress)) { + state.unlocked(); + } + return new UnlockExchange(new UnlockResponse(SetupAccess.UNLOCKED, session.expiresAt()), + SetupAccessCookie.create(session, context.secureCookie(), clock)); + } finally { + Arrays.fill(code, '\0'); + } + } + + public synchronized boolean permits(String token) { + return !requiresUnlock() || unlock.permits(token); + } + + public synchronized boolean permits(String token, HttpServletRequest request) { + return !requiresUnlock(request) || unlock.permits(token); + } + + @Override + public void close() throws IOException { + unlock.close(); + } + + /** Successful response plus the opaque transport cookie; neither renders the token. */ + public record UnlockExchange(UnlockResponse response, ResponseCookie cookie) { + @Override + public String toString() { + return "UnlockExchange[response=unlocked, cookie=redacted]"; + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupRequestSecurityPolicy.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupRequestSecurityPolicy.java new file mode 100644 index 0000000000..9e7c6f64c7 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupRequestSecurityPolicy.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.security; + +import jakarta.servlet.http.HttpServletRequest; +import java.util.List; + +/** Conservative direct-socket policy; forwarding headers are untrusted without an explicit trust boundary. */ +public final class SetupRequestSecurityPolicy { + private static final List FORWARDING_HEADERS = List.of( + "Forwarded", "X-Forwarded-For", "X-Forwarded-Proto", "X-Forwarded-Host"); + + public boolean hasUntrustedForwarding(HttpServletRequest request) { + return FORWARDING_HEADERS.stream().anyMatch(name -> request.getHeader(name) != null); + } + + public boolean secureCookie(HttpServletRequest request) { + return inspect(request).secureCookie(); + } + + public RequestContext inspect(HttpServletRequest request) { + RequestPath path = hasUntrustedForwarding(request) + ? RequestPath.UNTRUSTED_FORWARDED : RequestPath.DIRECT; + TransportSecurity transport = request.isSecure() + ? TransportSecurity.SECURE : TransportSecurity.CLEAR; + return new RequestContext(request.getRemoteAddr(), path, transport); + } + + /** Security-relevant request facts derived without trusting proxy-provided values. */ + public record RequestContext(String remoteAddress, RequestPath path, TransportSecurity transport) { + boolean requiresProofOnLoopback() { + return path == RequestPath.UNTRUSTED_FORWARDED; + } + + boolean secureCookie() { + return path == RequestPath.DIRECT && transport == TransportSecurity.SECURE; + } + } + + /** Whether the socket request arrived directly or carries untrusted forwarding metadata. */ + public enum RequestPath { + DIRECT, + UNTRUSTED_FORWARDED + } + + /** Security of the direct servlet transport, independent of forwarding metadata. */ + public enum TransportSecurity { + CLEAR, + SECURE + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupUnlockRejected.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupUnlockRejected.java index a6d2400995..0fde9a3f65 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupUnlockRejected.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupUnlockRejected.java @@ -28,7 +28,7 @@ public final class SetupUnlockRejected extends IllegalStateException { private final Reason reason; - SetupUnlockRejected(Reason reason) { + public SetupUnlockRejected(Reason reason) { super(safeMessage(reason)); this.reason = reason; } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupWriteAccessFilter.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupWriteAccessFilter.java new file mode 100644 index 0000000000..37480094b1 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SetupWriteAccessFilter.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.security; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.Cookie; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Clock; +import org.apache.hertzbeat.common.util.JsonUtil; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupErrorResponse; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.web.filter.OncePerRequestFilter; + +/** Requires the opaque unlock cookie for remote setup mutations. */ +public final class SetupWriteAccessFilter extends OncePerRequestFilter { + private final SetupHttpUnlockService unlock; + private final Clock clock; + + public SetupWriteAccessFilter(SetupHttpUnlockService unlock, Clock clock) { + this.unlock = unlock; + this.clock = clock; + } + + @Override + protected boolean shouldNotFilter(HttpServletRequest request) { + if (!unlock.requiresUnlock(request)) { + return true; + } + String path = request.getServletPath(); + return !path.startsWith("/api/setup/") + || SetupApiContract.STATUS_PATH.equals(path) + || SetupApiContract.UNLOCK_PATH.equals(path); + } + + @Override + protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain chain) + throws IOException, ServletException { + if (unlock.permits(cookie(request), request)) { + chain.doFilter(request, response); + return; + } + response.setStatus(HttpServletResponse.SC_FORBIDDEN); + response.setHeader(HttpHeaders.CACHE_CONTROL, "no-store"); + response.setCharacterEncoding(StandardCharsets.UTF_8.name()); + response.setContentType(MediaType.APPLICATION_JSON_VALUE); + response.getWriter().write(JsonUtil.toJson( + new SetupErrorResponse(SetupErrorCode.SETUP_LOCKED, clock.instant()))); + } + + private static String cookie(HttpServletRequest request) { + if (request.getCookies() == null) { + return null; + } + for (Cookie cookie : request.getCookies()) { + if (SetupAccessCookie.NAME.equals(cookie.getName())) { + return cookie.getValue(); + } + } + return null; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/unattended/SetupPasswordFileLoader.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/unattended/SetupPasswordFileLoader.java new file mode 100644 index 0000000000..a9279c9015 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/unattended/SetupPasswordFileLoader.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.unattended; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.springframework.core.env.Environment; + +/** Loads unattended passwords exclusively from bounded owner-only files. */ +public final class SetupPasswordFileLoader { + private static final int MAX_PASSWORD_BYTES = 16_384; + + public Password read(Path path) { + Path normalized = path.toAbsolutePath().normalize(); + byte[] encoded = null; + char[] decoded = null; + try { + if (!SecureSetupFile.isOwnerOnlyRegularFile(normalized)) { + throw new IllegalStateException("Setup password file is unavailable"); + } + long size = Files.size(normalized); + if (size <= 0 || size > MAX_PASSWORD_BYTES) { + throw new IllegalStateException("Setup password file size is invalid"); + } + encoded = Files.readAllBytes(normalized); + CharBuffer buffer = StandardCharsets.UTF_8.newDecoder().decode(ByteBuffer.wrap(encoded)); + decoded = new char[buffer.remaining()]; + buffer.get(decoded); + int length = withoutLineEnding(decoded); + if (length == 0) { + throw new IllegalStateException("Setup password file is empty"); + } + Password password = new Password(Arrays.copyOf(decoded, length)); + return password; + } catch (IOException failure) { + throw new IllegalStateException("Setup password file is unavailable"); + } finally { + if (encoded != null) { + Arrays.fill(encoded, (byte) 0); + } + if (decoded != null) { + Arrays.fill(decoded, '\0'); + } + } + } + + public static Path requireFilePath(Environment environment, String prefix) { + if (environment.getProperty(prefix + ".password") != null) { + throw new IllegalStateException("Plain setup password configuration is forbidden"); + } + String file = environment.getProperty(prefix + ".password-file"); + if (file == null || file.isBlank()) { + throw new IllegalStateException("Setup password file is required"); + } + return Path.of(file); + } + + private static int withoutLineEnding(char[] value) { + int length = value.length; + if (length > 0 && value[length - 1] == '\n') { + length--; + } + if (length > 0 && value[length - 1] == '\r') { + length--; + } + return length; + } + + /** Scoped password buffer; callers receive copies and must close the owner. */ + public static final class Password implements AutoCloseable { + private final char[] value; + + private Password(char[] value) { + this.value = value; + } + + public char[] copy() { + return value.clone(); + } + + public SecretValue secretValue() { + return SecretValue.of(value); + } + + @Override + public void close() { + Arrays.fill(value, '\0'); + } + + @Override + public String toString() { + return "Password[redacted]"; + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializer.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializer.java new file mode 100644 index 0000000000..2dfaa62b92 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializer.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.unattended; + +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.StatusResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransitionScheduler; +import org.apache.hertzbeat.manager.setup.workflow.HeadlessSetupWorkflow; +import org.springframework.core.env.Environment; + +/** Idempotent headless driver over the same public workflow and validators used by the browser. */ +public final class UnattendedSetupInitializer { + public static final String ENABLED_PROPERTY = "hertzbeat.setup.unattended.enabled"; + private static final String METADATA = "hertzbeat.setup.metadata"; + private static final String TELEMETRY = "hertzbeat.setup.telemetry"; + private static final String ADMINISTRATOR = "hertzbeat.setup.administrator"; + private static final String ACKNOWLEDGED_WARNINGS = + "hertzbeat.setup.unattended.acknowledged-warnings"; + private final HeadlessSetupWorkflow workflow; + private final Environment environment; + private final SetupPasswordFileLoader passwords; + private final Optional transitions; + + public UnattendedSetupInitializer( + HeadlessSetupWorkflow workflow, Environment environment, SetupPasswordFileLoader passwords) { + this(workflow, environment, passwords, Optional.empty()); + } + + public UnattendedSetupInitializer(HeadlessSetupWorkflow workflow, Environment environment, + SetupPasswordFileLoader passwords, + Optional transitions) { + this.workflow = workflow; + this.environment = environment; + this.passwords = passwords; + this.transitions = transitions; + } + + public void initialize() { + if (!environment.getProperty(ENABLED_PROPERTY, Boolean.class, false)) { + return; + } + StatusResponse status = workflow.status(); + switch (status.phase()) { + case CONFIGURATION_REQUIRED -> configure(status); + case ADMINISTRATOR_REQUIRED -> createAdministrator(); + case OPTIONAL_CONFIGURATION -> complete(); + case COMPLETE, EXTERNAL_APPLY_REQUIRED, APPLICATION_STARTING, + RECOVERY_REQUIRED, MIGRATION_IN_PROGRESS -> { + // A restart, external operator action, or recovery must converge state before another write. + } + default -> throw new IllegalStateException("Unsupported unattended setup phase"); + } + } + + private void configure(StatusResponse status) { + try (SetupPasswordFileLoader.Password metadataPassword = passwords.read( + SetupPasswordFileLoader.requireFilePath(environment, METADATA)); + SecretValue metadataSecret = metadataPassword.secretValue()) { + HeadlessSetupWorkflow.Metadata metadata = new HeadlessSetupWorkflow.Metadata( + MetadataDatabaseKind.valueOf(required(METADATA + ".kind").toUpperCase()), + required(METADATA + ".jdbc-url"), required(METADATA + ".username"), metadataSecret); + configureTelemetry(status, metadata); + } + } + + private void configureTelemetry(StatusResponse status, HeadlessSetupWorkflow.Metadata metadata) { + String username = environment.getProperty(TELEMETRY + ".username"); + String passwordFile = environment.getProperty(TELEMETRY + ".password-file"); + if (username == null && passwordFile == null) { + rejectPlainPassword(TELEMETRY); + configure(status, metadata, telemetry(Optional.empty(), Optional.empty())); + return; + } + Path path = SetupPasswordFileLoader.requireFilePath(environment, TELEMETRY); + try (SetupPasswordFileLoader.Password password = passwords.read(path); + SecretValue secret = password.secretValue()) { + configure(status, metadata, telemetry( + Optional.of(required(TELEMETRY + ".username")), Optional.of(secret))); + } + } + + private HeadlessSetupWorkflow.Telemetry telemetry( + Optional username, Optional password) { + return new HeadlessSetupWorkflow.Telemetry( + required(TELEMETRY + ".grpc-endpoints"), required(TELEMETRY + ".http-endpoint"), + required(TELEMETRY + ".database"), username, password); + } + + private void configure(StatusResponse status, HeadlessSetupWorkflow.Metadata metadata, + HeadlessSetupWorkflow.Telemetry telemetry) { + var response = workflow.configure( + new HeadlessSetupWorkflow.RequiredConfiguration(status.applyMode(), metadata, telemetry)); + if (response.phase() == SetupPhase.APPLICATION_STARTING) { + transitions.ifPresent(SetupRuntimeTransitionScheduler::configurationApplied); + } + } + + private void createAdministrator() { + Path path = SetupPasswordFileLoader.requireFilePath(environment, ADMINISTRATOR); + try (SetupPasswordFileLoader.Password password = passwords.read(path); + SecretValue secret = password.secretValue()) { + workflow.createAdministrator(required(ADMINISTRATOR + ".username"), secret); + } + complete(); + } + + private void complete() { + workflow.complete(acknowledgedWarnings()); + transitions.ifPresent(SetupRuntimeTransitionScheduler::installationCompleted); + } + + private List acknowledgedWarnings() { + String configured = environment.getProperty(ACKNOWLEDGED_WARNINGS, ""); + if (configured.isBlank()) { + return List.of(); + } + return Arrays.stream(configured.split(",")) + .map(String::trim) + .filter(value -> !value.isEmpty()) + .map(value -> SetupWarningCode.valueOf(value.toUpperCase(Locale.ROOT))) + .toList(); + } + + private String required(String key) { + String value = environment.getProperty(key); + if (value == null || value.isBlank()) { + throw new IllegalStateException("Required unattended setup property is missing"); + } + return value; + } + + private void rejectPlainPassword(String prefix) { + if (environment.getProperty(prefix + ".password") != null) { + throw new IllegalStateException("Plain setup password configuration is forbidden"); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflow.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflow.java new file mode 100644 index 0000000000..6d1d527774 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflow.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.time.Clock; +import java.util.Arrays; +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.AdministratorRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.AdministratorResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.CompleteRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.CompleteResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OperationResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionalConfigurationSummary; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.StatusResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.UnlockRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.UnlockResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidateRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationSection; +import org.apache.hertzbeat.manager.setup.api.SetupApiException; +import org.apache.hertzbeat.manager.setup.api.SetupWorkflow; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; +import org.apache.hertzbeat.manager.setup.identity.AdministratorCredentials; +import org.apache.hertzbeat.manager.setup.identity.BootstrapIdentityConflict; +import org.apache.hertzbeat.manager.setup.identity.IdentityInitializationService; +import org.springframework.http.HttpStatus; + +/** Cohesive setup state-machine facade; transport and persistence remain in dedicated collaborators. */ +public final class DefaultSetupWorkflow implements SetupWorkflow { + private final SetupRuntimeState state; + private final SetupRequestValidator validator; + private final SetupConfigurationCoordinator configuration; + private final SetupOperationRegistry operations; + private final ManagedConfigCapability capability; + private final Optional identities; + private final Optional completion; + private final SetupOptionsCoordinator options; + private final Clock clock; + private final SetupMutationSerializer mutations; + + public DefaultSetupWorkflow(SetupRuntimeState state, SetupRequestValidator validator, + SetupConfigurationCoordinator configuration, SetupOperationRegistry operations, + ManagedConfigCapability capability, Optional identities, + Optional completion, SetupOptionsCoordinator options, + Clock clock, SetupMutationSerializer mutations) { + this.state = state; + this.validator = validator; + this.configuration = configuration; + this.operations = operations; + this.capability = capability; + this.identities = identities; + this.completion = completion; + this.options = options; + this.clock = clock; + this.mutations = mutations; + } + + @Override + public StatusResponse status() { + return state.status(); + } + + @Override + public UnlockResponse unlock(UnlockRequest request) { + requireWritable(); + throw new SetupApiException(SetupErrorCode.SETUP_CODE_INVALID, HttpStatus.FORBIDDEN); + } + + @Override + public ValidationResponse validate(ValidateRequest request) { + requireWritable(); + return validator.validate(request); + } + + @Override + public ConfigurationResponse configure(ConfigurationRequest request) { + return mutations.execute(() -> configureMutation(request)); + } + + private ConfigurationResponse configureMutation(ConfigurationRequest request) { + requireWritable(); + state.ensurePhase(SetupPhase.CONFIGURATION_REQUIRED); + requireValid(new ValidateRequest(ValidationSection.METADATA_DATABASE, + request.managementDatabase(), null, null, null)); + requireValid(new ValidateRequest(ValidationSection.TELEMETRY_STORE, + null, request.telemetryStore(), null, null)); + ConfigurationResponse response = configuration.configure(request, capability); + state.configurationApplied(response.operationId(), response.phase()); + return response; + } + + @Override + public OperationResponse operation(String operationId) { + return operations.get(operationId); + } + + @Override + public AdministratorResponse createAdministrator(AdministratorRequest request) { + return mutations.execute(() -> createAdministratorMutation(request)); + } + + private AdministratorResponse createAdministratorMutation(AdministratorRequest request) { + requireWritable(); + state.ensurePhase(SetupPhase.ADMINISTRATOR_REQUIRED); + char[] password = request.password().toCharArray(); + try { + identities.orElseThrow(SetupWorkflowConflict::new) + .createFirstAdministrator(new AdministratorCredentials(request.username(), password)); + } catch (BootstrapIdentityConflict conflict) { + throw new SetupApiException(SetupErrorCode.ADMINISTRATOR_ALREADY_CONFIGURED, HttpStatus.CONFLICT); + } finally { + Arrays.fill(password, '\0'); + } + state.administratorCreated(request.username()); + return new AdministratorResponse(request.username(), SetupPhase.OPTIONAL_CONFIGURATION); + } + + @Override + public OptionsResponse configureOptions(OptionsRequest request) { + return mutations.execute(() -> configureOptionsMutation(request)); + } + + private OptionsResponse configureOptionsMutation(OptionsRequest request) { + requireWritable(); + state.ensurePhase(SetupPhase.OPTIONAL_CONFIGURATION); + if (request.publicAccess() != null) { + requireValid(new ValidateRequest(ValidationSection.PUBLIC_ACCESS, + null, null, request.publicAccess(), null)); + } + if (request.mail() != null) { + requireValid(new ValidateRequest(ValidationSection.MAIL, + null, null, null, request.mail())); + } + options.persist(request); + OptionalConfigurationSummary summary = new OptionalConfigurationSummary( + request.publicAccess() != null && hasText(request.publicAccess().publicBaseUrl()), + request.publicAccess() != null && hasText(request.publicAccess().serverOtlpHttpEndpoint()), + request.publicAccess() != null && hasText(request.publicAccess().serverOtlpGrpcEndpoint()), + request.retention() != null, request.mail() != null); + state.optionsConfigured(summary, + SetupWarningPolicy.INSTANCE.evaluate(state.managementDatabaseKind(), request)); + return new OptionsResponse(summary.publicAccessConfigured(), summary.serverOtlpHttpConfigured(), + summary.serverOtlpGrpcConfigured(), summary.retentionConfigured(), summary.mailConfigured(), + SetupPhase.OPTIONAL_CONFIGURATION); + } + + @Override + public ExportResponse prepareExport(ExportRequest request) { + requireWritable(); + return switch (request.format()) { + case YAML -> new ExportResponse("hertzbeat-setup.yml", "application/yaml"); + case ENV -> new ExportResponse("hertzbeat-setup.env", "text/plain"); + case KUBERNETES_SECRET -> new ExportResponse("hertzbeat-setup-secret.yml", "application/yaml"); + }; + } + + @Override + public CompleteResponse complete(CompleteRequest request) { + return mutations.execute(() -> completeMutation(request)); + } + + private CompleteResponse completeMutation(CompleteRequest request) { + requireWritable(); + state.ensurePhase(SetupPhase.OPTIONAL_CONFIGURATION); + if (request.expectedPhase() != SetupPhase.OPTIONAL_CONFIGURATION) { + throw new SetupWorkflowConflict(); + } + if (!request.acknowledgedWarnings().containsAll(state.pendingWarnings())) { + throw new SetupApiException(SetupErrorCode.OPERATION_CONFLICT, HttpStatus.CONFLICT); + } + String username = state.administratorUsername(); + if (username == null) { + throw new SetupWorkflowConflict(); + } + completion.orElseThrow(SetupWorkflowConflict::new).completeInstallation(); + state.complete(); + CompleteResponse response = new CompleteResponse(SetupPhase.COMPLETE, clock.instant(), "/login", username); + return response; + } + + private void requireWritable() { + if (state.phase() == SetupPhase.COMPLETE) { + throw new SetupApiException(SetupErrorCode.SETUP_COMPLETE, HttpStatus.GONE); + } + } + + private void requireValid(ValidateRequest request) { + ValidationResponse response = validator.validate(request); + if (!response.valid()) { + throw new SetupApiException(response.errorCode(), HttpStatus.BAD_REQUEST); + } + } + + private static boolean hasText(String value) { + return value != null && !value.isBlank(); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/GreptimeHttpConnectionProbe.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/GreptimeHttpConnectionProbe.java new file mode 100644 index 0000000000..723e078c3d --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/GreptimeHttpConnectionProbe.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.net.URI; +import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Arrays; +import java.util.Base64; +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.config.SecretValue; + +/** Bounded Greptime HTTP health/authentication adapter. */ +public final class GreptimeHttpConnectionProbe implements TelemetryConnectionProbe { + private final Duration timeout; + private final HttpClient client; + + public GreptimeHttpConnectionProbe(Duration timeout) { + this(timeout, HttpClient.newBuilder().connectTimeout(timeout).build()); + } + + GreptimeHttpConnectionProbe(Duration timeout, HttpClient client) { + this.timeout = timeout; + this.client = client; + } + + @Override + public Optional probe(TelemetryConnectionProbe.Request configuration) { + try { + String endpoint = configuration.httpEndpoint().replaceAll("/+$", "") + "/v1/sql?db=" + + URLEncoder.encode(configuration.database(), StandardCharsets.UTF_8); + HttpRequest.Builder request = HttpRequest.newBuilder(URI.create(endpoint)) + .timeout(timeout).header("Accept", "application/json") + .header("Content-Type", "application/x-www-form-urlencoded") + .POST(HttpRequest.BodyPublishers.ofString("sql=SELECT%201", StandardCharsets.US_ASCII)); + if (configuration.username().isPresent()) { + request.header("Authorization", basicAuthorization( + configuration.username().orElseThrow(), configuration.password().orElseThrow())); + } + int status = client.send(request.build(), HttpResponse.BodyHandlers.discarding()).statusCode(); + return status >= 200 && status < 300 + ? Optional.empty() : Optional.of(SetupErrorCode.TELEMETRY_CONNECTION_FAILED); + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + return Optional.of(SetupErrorCode.TELEMETRY_CONNECTION_FAILED); + } catch (Exception failure) { + return Optional.of(SetupErrorCode.TELEMETRY_CONNECTION_FAILED); + } + } + + private static String basicAuthorization(String username, SecretValue password) { + byte[] usernameBytes = username.getBytes(StandardCharsets.UTF_8); + char[] passwordCharacters = password.copy(); + ByteBuffer passwordBytes = StandardCharsets.UTF_8.encode(CharBuffer.wrap(passwordCharacters)); + byte[] credentials = new byte[usernameBytes.length + 1 + passwordBytes.remaining()]; + byte[] encoded = null; + try { + System.arraycopy(usernameBytes, 0, credentials, 0, usernameBytes.length); + credentials[usernameBytes.length] = ':'; + passwordBytes.get(credentials, usernameBytes.length + 1, passwordBytes.remaining()); + encoded = Base64.getEncoder().encode(credentials); + // HttpRequest headers require a String; retain only the encoded header at this JDK boundary. + return "Basic " + new String(encoded, StandardCharsets.US_ASCII); + } finally { + Arrays.fill(usernameBytes, (byte) 0); + Arrays.fill(passwordCharacters, '\0'); + if (passwordBytes.hasArray()) { + Arrays.fill(passwordBytes.array(), (byte) 0); + } + Arrays.fill(credentials, (byte) 0); + if (encoded != null) { + Arrays.fill(encoded, (byte) 0); + } + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinator.java new file mode 100644 index 0000000000..629d02134d --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinator.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.StatusResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiException; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.apache.hertzbeat.manager.setup.identity.AdministratorCredentials; +import org.apache.hertzbeat.manager.setup.identity.BootstrapIdentityConflict; +import org.apache.hertzbeat.manager.setup.identity.IdentityInitializationService; +import org.springframework.http.HttpStatus; + +/** Executes non-HTTP setup commands while retaining clearable secret ownership. */ +public final class HeadlessSetupCoordinator implements HeadlessSetupWorkflow { + private final SetupRuntimeState state; + private final SetupRequestValidator validator; + private final SetupConfigurationCoordinator configuration; + private final ManagedConfigCapability capability; + private final Optional identities; + private final Optional completion; + private final SetupMutationSerializer mutations; + + public HeadlessSetupCoordinator(SetupRuntimeState state, SetupRequestValidator validator, + SetupConfigurationCoordinator configuration, + ManagedConfigCapability capability, + Optional identities, + Optional completion, + SetupMutationSerializer mutations) { + this.state = state; + this.validator = validator; + this.configuration = configuration; + this.capability = capability; + this.identities = identities; + this.completion = completion; + this.mutations = mutations; + } + + @Override + public StatusResponse status() { + return state.status(); + } + + @Override + public ConfigurationResponse configure(RequiredConfiguration request) { + return mutations.execute(() -> configureMutation(request)); + } + + private ConfigurationResponse configureMutation(RequiredConfiguration request) { + requireWritable(); + state.ensurePhase(SetupPhase.CONFIGURATION_REQUIRED); + validator.validate(request.metadata()); + validator.validate(request.telemetry()); + ConfigurationResponse response = configuration.configure( + request, SetupConfigurationMapper.map(request), capability); + state.configurationApplied(response.operationId(), response.phase()); + return response; + } + + @Override + public void createAdministrator(String username, SecretValue password) { + mutations.execute(() -> createAdministratorMutation(username, password)); + } + + private void createAdministratorMutation(String username, SecretValue password) { + requireWritable(); + state.ensurePhase(SetupPhase.ADMINISTRATOR_REQUIRED); + char[] clear = password.copy(); + try (AdministratorCredentials credentials = new AdministratorCredentials(username, clear)) { + identities.orElseThrow(SetupWorkflowConflict::new).createFirstAdministrator(credentials); + } catch (BootstrapIdentityConflict conflict) { + throw new SetupApiException(SetupErrorCode.ADMINISTRATOR_ALREADY_CONFIGURED, HttpStatus.CONFLICT); + } finally { + Arrays.fill(clear, '\0'); + } + state.administratorCreated(username); + } + + @Override + public void complete(List acknowledgedWarnings) { + mutations.execute(() -> completeMutation(acknowledgedWarnings)); + } + + private void completeMutation(List acknowledgedWarnings) { + requireWritable(); + state.ensurePhase(SetupPhase.OPTIONAL_CONFIGURATION); + if (!acknowledgedWarnings.containsAll(state.pendingWarnings())) { + throw new SetupApiException(SetupErrorCode.OPERATION_CONFLICT, HttpStatus.CONFLICT); + } + if (state.administratorUsername() == null) { + throw new SetupWorkflowConflict(); + } + completion.orElseThrow(SetupWorkflowConflict::new).completeInstallation(); + state.complete(); + } + + private void requireWritable() { + if (state.phase() == SetupPhase.COMPLETE) { + throw new SetupApiException(SetupErrorCode.SETUP_COMPLETE, HttpStatus.GONE); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupWorkflow.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupWorkflow.java new file mode 100644 index 0000000000..ae1f7ba98f --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupWorkflow.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; +import java.util.List; +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.StatusResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; +import org.apache.hertzbeat.manager.setup.config.SecretValue; + +/** Internal headless command boundary that keeps file passwords out of immutable DTO strings. */ +public interface HeadlessSetupWorkflow { + StatusResponse status(); + + ConfigurationResponse configure(RequiredConfiguration configuration); + + void createAdministrator(String username, SecretValue password); + + void complete(List acknowledgedWarnings); + + /** Required managed configuration with secret values kept in clearable owners. */ + record RequiredConfiguration(ApplyMode applyMode, Metadata metadata, Telemetry telemetry) { + public RequiredConfiguration { + Objects.requireNonNull(applyMode, "applyMode"); + Objects.requireNonNull(metadata, "metadata"); + Objects.requireNonNull(telemetry, "telemetry"); + } + } + + /** Headless metadata settings. */ + record Metadata(MetadataDatabaseKind kind, String jdbcUrl, String username, SecretValue password) { + } + + /** Headless Greptime settings. */ + record Telemetry(String grpcEndpoints, String httpEndpoint, String database, + Optional username, Optional password) { + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JakartaMailConnectionProbe.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JakartaMailConnectionProbe.java new file mode 100644 index 0000000000..1d6c8cddda --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JakartaMailConnectionProbe.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import jakarta.mail.Session; +import jakarta.mail.Transport; +import java.time.Duration; +import java.util.Optional; +import java.util.Properties; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Bounded mail transport connection/authentication adapter; it never sends a message. */ +public final class JakartaMailConnectionProbe implements MailConnectionProbe { + private final Duration timeout; + + public JakartaMailConnectionProbe(Duration timeout) { + this.timeout = timeout; + } + + @Override + public Optional probe(MailConfiguration configuration) { + String protocol = configuration.security() == MailSecurity.TLS ? "smtps" : "smtp"; + Properties properties = properties(protocol, configuration.security()); + Session session = Session.getInstance(properties); + try (Transport transport = session.getTransport(protocol)) { + transport.connect(configuration.host(), configuration.port(), + configuration.username(), configuration.password()); + return Optional.empty(); + } catch (Exception failure) { + return Optional.of(SetupErrorCode.MAIL_CONNECTION_FAILED); + } + } + + private Properties properties(String protocol, MailSecurity security) { + int millis = Math.toIntExact(timeout.toMillis()); + Properties properties = new Properties(); + properties.setProperty("mail." + protocol + ".connectiontimeout", Integer.toString(millis)); + properties.setProperty("mail." + protocol + ".timeout", Integer.toString(millis)); + properties.setProperty("mail." + protocol + ".writetimeout", Integer.toString(millis)); + if (security == MailSecurity.STARTTLS) { + properties.setProperty("mail.smtp.starttls.enable", "true"); + properties.setProperty("mail.smtp.starttls.required", "true"); + } + return properties; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataConnectionProbe.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataConnectionProbe.java new file mode 100644 index 0000000000..b15fccb2cc --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataConnectionProbe.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.util.Arrays; +import java.util.Locale; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Bounded JDBC connection, dialect/charset, schema access, and temporary DDL/DML probe. */ +public final class JdbcMetadataConnectionProbe implements MetadataConnectionProbe { + private static final int MAX_CONCURRENT_PROBES = 2; + private static final int MAX_QUEUED_PROBES = 4; + private static final ThreadPoolExecutor SHARED_EXECUTOR = new ThreadPoolExecutor( + MAX_CONCURRENT_PROBES, MAX_CONCURRENT_PROBES, 0L, TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(MAX_QUEUED_PROBES), task -> { + Thread thread = new Thread(task, "setup-metadata-probe"); + thread.setDaemon(true); + return thread; + }, new ThreadPoolExecutor.AbortPolicy()); + private final Duration timeout; + private final ThreadPoolExecutor executor; + private final JdbcConnector connector; + + public JdbcMetadataConnectionProbe(Duration timeout) { + this(timeout, SHARED_EXECUTOR, (url, username, password) -> + DriverManager.getConnection(url, username, new String(password))); + } + + JdbcMetadataConnectionProbe(Duration timeout, ThreadPoolExecutor executor, JdbcConnector connector) { + this.timeout = timeout; + this.executor = executor; + this.connector = connector; + } + + @Override + public Optional probe(MetadataConnectionProbe.Request configuration) { + AtomicReference activeConnection = new AtomicReference<>(); + Future> future; + try { + future = executor.submit(() -> validate(configuration, activeConnection)); + } catch (RejectedExecutionException overload) { + return Optional.of(SetupErrorCode.METADATA_CONNECTION_FAILED); + } + try { + return future.get(timeout.toMillis(), TimeUnit.MILLISECONDS); + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + return Optional.of(SetupErrorCode.METADATA_CONNECTION_FAILED); + } catch (ExecutionException | TimeoutException failure) { + future.cancel(true); + close(activeConnection.get()); + return Optional.of(SetupErrorCode.METADATA_CONNECTION_FAILED); + } + } + + private Optional validate(MetadataConnectionProbe.Request configuration, + AtomicReference activeConnection) { + char[] password = configuration.password().copy(); + try (Connection connection = connector.connect(configuration.jdbcUrl(), + configuration.username(), password)) { + activeConnection.set(connection); + connection.setNetworkTimeout(Runnable::run, Math.toIntExact(timeout.toMillis())); + Optional compatibility = validateCompatibility(connection, configuration.kind()); + if (compatibility.isPresent()) { + return compatibility; + } + return validatePrivileges(connection); + } catch (SQLException | RuntimeException failure) { + return Optional.of(SetupErrorCode.METADATA_CONNECTION_FAILED); + } finally { + Arrays.fill(password, '\0'); + activeConnection.set(null); + } + } + + private Optional validateCompatibility(Connection connection, + MetadataDatabaseKind expected) throws SQLException { + DatabaseMetaData metadata = connection.getMetaData(); + String product = metadata.getDatabaseProductName().toLowerCase(Locale.ROOT); + boolean matches = switch (expected) { + case H2 -> product.contains("h2"); + case MYSQL -> product.contains("mysql"); + case POSTGRESQL -> product.contains("postgresql"); + }; + if (!matches || !utf8Compatible(connection, expected)) { + return Optional.of(SetupErrorCode.METADATA_SCHEMA_MISMATCH); + } + try (ResultSet ignored = metadata.getSchemas()) { + // Opening the schema projection verifies that metadata visibility is available. + } + return Optional.empty(); + } + + private boolean utf8Compatible(Connection connection, MetadataDatabaseKind kind) throws SQLException { + String sql = switch (kind) { + case MYSQL -> "SELECT @@character_set_database"; + case POSTGRESQL -> "SHOW server_encoding"; + case H2 -> null; + }; + if (sql == null) { + return true; + } + try (Statement statement = connection.createStatement()) { + statement.setQueryTimeout(Math.max(1, Math.toIntExact(timeout.toSeconds()))); + try (ResultSet result = statement.executeQuery(sql)) { + return result.next() && result.getString(1).toLowerCase(Locale.ROOT).startsWith("utf8"); + } + } + } + + Optional validatePrivileges(Connection connection) { + String table = "HZB_SETUP_PROBE_" + UUID.randomUUID().toString().replace("-", ""); + boolean created = false; + try { + connection.setAutoCommit(false); + try (Statement statement = connection.createStatement()) { + statement.setQueryTimeout(Math.max(1, Math.toIntExact(timeout.toSeconds()))); + statement.execute("CREATE TABLE " + table + + " (probe_id INTEGER NOT NULL PRIMARY KEY, probe_value VARCHAR(32) NOT NULL)"); + created = true; + statement.executeUpdate("INSERT INTO " + table + " VALUES (1, 'created')"); + statement.executeUpdate("UPDATE " + table + " SET probe_value = 'updated' WHERE probe_id = 1"); + try (ResultSet result = statement.executeQuery("SELECT probe_value FROM " + table + + " WHERE probe_id = 1")) { + if (!result.next() || !"updated".equals(result.getString(1))) { + throw new SQLException("Temporary probe value was not readable"); + } + } + statement.executeUpdate("DELETE FROM " + table + " WHERE probe_id = 1"); + statement.execute("DROP TABLE " + table); + created = false; + } + connection.commit(); + return Optional.empty(); + } catch (SQLException failure) { + rollback(connection); + if (created) { + drop(connection, table); + } + return Optional.of(SetupErrorCode.METADATA_INSUFFICIENT_PRIVILEGES); + } + } + + private static boolean drop(Connection connection, String table) { + try (Statement statement = connection.createStatement()) { + statement.execute("DROP TABLE " + table); + connection.commit(); + return true; + } catch (SQLException ignored) { + return false; + } + } + + private static void rollback(Connection connection) { + try { + connection.rollback(); + } catch (SQLException ignored) { + // Preserve the stable validation error. + } + } + + private static void close(Connection connection) { + if (connection != null) { + try { + connection.close(); + } catch (SQLException ignored) { + // Preserve the stable timeout error. + } + } + } + + @FunctionalInterface + interface JdbcConnector { + Connection connect(String url, String username, char[] password) throws SQLException; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MailConfigurationValidator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MailConfigurationValidator.java new file mode 100644 index 0000000000..d273ddacc3 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MailConfigurationValidator.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.List; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; +import org.apache.hertzbeat.manager.setup.workflow.MetadataConfigurationValidator.Validation; + +/** Validates mail credential pairing and reports an explicitly insecure transport. */ +final class MailConfigurationValidator { + Validation validate(MailConfiguration configuration) { + if (configuration.port() > 65_535) { + return Validation.failed(SetupErrorCode.MAIL_CONNECTION_FAILED); + } + boolean username = configuration.username() != null && !configuration.username().isBlank(); + boolean password = configuration.password() != null && !configuration.password().isBlank(); + if (username != password) { + return Validation.failed(SetupErrorCode.MAIL_CONNECTION_FAILED); + } + List warnings = configuration.security() == MailSecurity.NONE + ? List.of(SetupWarningCode.MAIL_SECURITY_NONE) : List.of(); + return new Validation(true, null, warnings); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MailConnectionProbe.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MailConnectionProbe.java new file mode 100644 index 0000000000..f4705b5eac --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MailConnectionProbe.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Injectable mail transport connection/authentication test boundary. */ +@FunctionalInterface +public interface MailConnectionProbe { + Optional probe(MailConfiguration configuration); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataConfigurationValidator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataConfigurationValidator.java new file mode 100644 index 0000000000..491d635c56 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataConfigurationValidator.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.List; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; + +/** Validates metadata database type/address consistency before a connectivity probe. */ +final class MetadataConfigurationValidator { + Validation validate(MetadataDatabaseConfiguration configuration) { + return validate(configuration.kind(), configuration.jdbcUrl()); + } + + Validation validate(MetadataDatabaseKind kind, String jdbcUrl) { + String expectedPrefix = switch (kind) { + case H2 -> "jdbc:h2:"; + case MYSQL -> "jdbc:mysql:"; + case POSTGRESQL -> "jdbc:postgresql:"; + }; + if (!jdbcUrl.startsWith(expectedPrefix)) { + return Validation.failed(SetupErrorCode.METADATA_KIND_UNSUPPORTED); + } + return Validation.success(); + } + + record Validation(boolean valid, SetupErrorCode errorCode, List warnings) { + static Validation success() { + return new Validation(true, null, List.of()); + } + + static Validation failed(SetupErrorCode code) { + return new Validation(false, code, List.of()); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataConnectionProbe.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataConnectionProbe.java new file mode 100644 index 0000000000..eb66ff90ae --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataConnectionProbe.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.config.SecretValue; + +/** Injectable live metadata connection and privilege validation boundary. */ +@FunctionalInterface +public interface MetadataConnectionProbe { + Optional probe(Request configuration); + + default Optional probe(MetadataDatabaseConfiguration configuration) { + try (Request request = new Request(configuration.kind(), configuration.jdbcUrl(), + configuration.username(), SecretValue.of(configuration.password()))) { + return probe(request); + } + } + + default Optional probe(HeadlessSetupWorkflow.Metadata configuration) { + try (Request request = new Request(configuration.kind(), configuration.jdbcUrl(), + configuration.username(), SecretValue.copyOf(configuration.password()))) { + return probe(request); + } + } + + /** Probe-scoped clearable credentials; the probe must not retain this request after return. */ + record Request(MetadataDatabaseKind kind, String jdbcUrl, String username, + SecretValue password) implements AutoCloseable { + @Override + public void close() { + password.close(); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/PublicAccessConfigurationValidator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/PublicAccessConfigurationValidator.java new file mode 100644 index 0000000000..abf91a0714 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/PublicAccessConfigurationValidator.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.PublicAccessConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; +import org.apache.hertzbeat.manager.setup.workflow.MetadataConfigurationValidator.Validation; + +/** Validates optional public endpoint URIs and reports plaintext exposure explicitly. */ +final class PublicAccessConfigurationValidator { + Validation validate(PublicAccessConfiguration configuration) { + List warnings = new ArrayList<>(); + for (String value : List.of(nullToEmpty(configuration.publicBaseUrl()), + nullToEmpty(configuration.serverOtlpHttpEndpoint()))) { + if (value.isEmpty()) { + continue; + } + URI uri; + try { + uri = URI.create(value); + } catch (IllegalArgumentException failure) { + return Validation.failed(SetupErrorCode.PUBLIC_ADDRESS_INVALID); + } + if (uri.getHost() == null || !("http".equalsIgnoreCase(uri.getScheme()) + || "https".equalsIgnoreCase(uri.getScheme()))) { + return Validation.failed(SetupErrorCode.PUBLIC_ADDRESS_INVALID); + } + if ("http".equalsIgnoreCase(uri.getScheme())) { + warnings.add(SetupWarningCode.PUBLIC_ADDRESS_PLAINTEXT); + } + } + String grpc = configuration.serverOtlpGrpcEndpoint(); + if (grpc != null && !grpc.isBlank() && !validHostPort(grpc)) { + return Validation.failed(SetupErrorCode.PUBLIC_ADDRESS_INVALID); + } + return new Validation(true, null, List.copyOf(warnings)); + } + + private static boolean validHostPort(String value) { + int separator = value.lastIndexOf(':'); + if (separator < 1 || separator == value.length() - 1 || value.indexOf('/') >= 0 + || value.substring(0, separator).isBlank() || value.substring(0, separator).contains(" ")) { + return false; + } + try { + int port = Integer.parseInt(value.substring(separator + 1)); + return port > 0 && port <= 65_535; + } catch (NumberFormatException failure) { + return false; + } + } + + private static String nullToEmpty(String value) { + return value == null ? "" : value; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupCompletionCoordinator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupCompletionCoordinator.java new file mode 100644 index 0000000000..2c99ee56c5 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupCompletionCoordinator.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.IOException; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiException; +import org.apache.hertzbeat.manager.setup.installation.InstallationCompletionService; +import org.apache.hertzbeat.manager.setup.installation.InstallationFingerprint; +import org.apache.hertzbeat.manager.setup.installation.LocalInstallationFingerprintStore; +import org.springframework.http.HttpStatus; + +/** Commits installation closure and schedules the context transition off the request thread. */ +public final class SetupCompletionCoordinator { + private final LocalInstallationFingerprintStore fingerprints; + private final InstallationCompletionService installations; + + public SetupCompletionCoordinator(LocalInstallationFingerprintStore fingerprints, + InstallationCompletionService installations) { + this.fingerprints = fingerprints; + this.installations = installations; + } + + public void completeInstallation() { + InstallationFingerprint fingerprint; + try { + fingerprint = fingerprints.read().orElseGet(this::createFingerprint); + } catch (IOException failure) { + throw new SetupApiException(SetupErrorCode.CONFIG_WRITE_FAILED, HttpStatus.INTERNAL_SERVER_ERROR); + } + installations.complete(fingerprint); + } + + private InstallationFingerprint createFingerprint() { + try { + return fingerprints.create(); + } catch (IOException failure) { + try { + return fingerprints.read().orElseThrow(() -> failure); + } catch (IOException readFailure) { + throw new SetupApiException(SetupErrorCode.CONFIG_WRITE_FAILED, HttpStatus.INTERNAL_SERVER_ERROR); + } + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationCoordinator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationCoordinator.java new file mode 100644 index 0000000000..a25578f64c --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationCoordinator.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.IOException; +import org.apache.hertzbeat.manager.setup.api.SetupApiException; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigurationBundle; +import org.springframework.http.HttpStatus; + +/** Owns required-configuration application and its operation state transitions. */ +public final class SetupConfigurationCoordinator { + private final ManagedConfigurationTransaction transaction; + private final SetupOperationRegistry operations; + + public SetupConfigurationCoordinator( + ManagedConfigurationTransaction transaction, SetupOperationRegistry operations) { + this.transaction = transaction; + this.operations = operations; + } + + public ConfigurationResponse configure(ConfigurationRequest request, ManagedConfigCapability capability) { + if (request.expectedPhase() != SetupPhase.CONFIGURATION_REQUIRED + || request.applyMode() != capability.applyMode()) { + throw new SetupWorkflowConflict(); + } + String operationId = operations.begin(SetupPhase.CONFIGURATION_REQUIRED); + if (request.applyMode() == ApplyMode.EXTERNAL_APPLY) { + operations.finish(operationId, SetupOperationState.AWAITING_EXTERNAL_APPLY, + SetupPhase.EXTERNAL_APPLY_REQUIRED, null, true); + return response(operationId); + } + try { + return applyManaged(operationId, request); + } catch (IOException failure) { + operations.finish(operationId, SetupOperationState.FAILED, + SetupPhase.CONFIGURATION_REQUIRED, SetupErrorCode.CONFIG_WRITE_FAILED, false); + throw new SetupApiException(SetupErrorCode.CONFIG_WRITE_FAILED, HttpStatus.INTERNAL_SERVER_ERROR); + } + } + + public ConfigurationResponse configure(HeadlessSetupWorkflow.RequiredConfiguration request, + ManagedConfigurationBundle bundle, + ManagedConfigCapability capability) { + try (bundle) { + if (request.applyMode() != capability.applyMode()) { + throw new SetupWorkflowConflict(); + } + String operationId = operations.begin(SetupPhase.CONFIGURATION_REQUIRED); + if (request.applyMode() == ApplyMode.EXTERNAL_APPLY) { + operations.finish(operationId, SetupOperationState.AWAITING_EXTERNAL_APPLY, + SetupPhase.EXTERNAL_APPLY_REQUIRED, null, true); + return response(operationId); + } + try { + return applyManaged(operationId, bundle); + } catch (IOException failure) { + operations.finish(operationId, SetupOperationState.FAILED, + SetupPhase.CONFIGURATION_REQUIRED, SetupErrorCode.CONFIG_WRITE_FAILED, false); + throw new SetupApiException(SetupErrorCode.CONFIG_WRITE_FAILED, HttpStatus.INTERNAL_SERVER_ERROR); + } + } + } + + private ConfigurationResponse applyManaged(String operationId, ConfigurationRequest request) throws IOException { + try (ManagedConfigurationBundle bundle = SetupConfigurationMapper.map(request)) { + return applyManaged(operationId, bundle); + } + } + + private ConfigurationResponse applyManaged(String operationId, ManagedConfigurationBundle bundle) + throws IOException { + ManagedConfigurationTransaction.Outcome outcome = transaction.apply(bundle); + if (outcome == ManagedConfigurationTransaction.Outcome.RECOVERY_REQUIRED) { + operations.finish(operationId, SetupOperationState.FAILED, + SetupPhase.RECOVERY_REQUIRED, SetupErrorCode.CONFIG_RECOVERY_REQUIRED, false); + throw new SetupApiException(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, HttpStatus.CONFLICT); + } + if (outcome != ManagedConfigurationTransaction.Outcome.APPLIED) { + operations.finish(operationId, SetupOperationState.ROLLED_BACK, + SetupPhase.CONFIGURATION_REQUIRED, SetupErrorCode.CONFIG_WRITE_FAILED, false); + throw new SetupApiException(SetupErrorCode.CONFIG_WRITE_FAILED, HttpStatus.INTERNAL_SERVER_ERROR); + } + operations.finish(operationId, SetupOperationState.AWAITING_RESTART, + SetupPhase.APPLICATION_STARTING, null, false); + return response(operationId); + } + + private ConfigurationResponse response(String operationId) { + var operation = operations.get(operationId); + return new ConfigurationResponse(operation.operationId(), operation.state(), operation.phase(), + operation.nextPollAfterMillis(), operation.exportAvailable()); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationMapper.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationMapper.java new file mode 100644 index 0000000000..20de92d331 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationMapper.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreConfiguration; +import org.apache.hertzbeat.manager.setup.config.GreptimeEndpoints; +import org.apache.hertzbeat.manager.setup.config.GreptimeSettings; +import org.apache.hertzbeat.manager.setup.config.ManagedApplicationConfig; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigurationBundle; +import org.apache.hertzbeat.manager.setup.config.ManagedSecrets; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; + +/** Maps inputs into a coordinator-owned bundle whose secrets never alias caller-owned values. */ +final class SetupConfigurationMapper { + private SetupConfigurationMapper() { + } + + static ManagedConfigurationBundle map(ConfigurationRequest request) { + var metadata = request.managementDatabase(); + var telemetry = request.telemetryStore(); + GreptimeSettings telemetrySettings = telemetry.username() == null + ? GreptimeSettings.anonymous(endpoints(telemetry), telemetry.database()) + : GreptimeSettings.authenticated(endpoints(telemetry), telemetry.database(), telemetry.username()); + ManagedApplicationConfig application = new ManagedApplicationConfig( + new MetadataDatabaseSettings(metadata.kind(), metadata.jdbcUrl(), metadata.username()), + telemetrySettings); + SecretValue metadataPassword = SecretValue.of(metadata.password()); + ManagedSecrets secrets = telemetry.password() == null + ? ManagedSecrets.withoutTelemetryPassword(metadataPassword) + : ManagedSecrets.withTelemetryPassword(metadataPassword, SecretValue.of(telemetry.password())); + return new ManagedConfigurationBundle(application, secrets); + } + + static ManagedConfigurationBundle map(HeadlessSetupWorkflow.RequiredConfiguration request) { + var telemetry = request.telemetry(); + GreptimeEndpoints endpoints = new GreptimeEndpoints( + telemetry.grpcEndpoints(), telemetry.httpEndpoint()); + GreptimeSettings telemetrySettings = telemetry.username().isPresent() + ? GreptimeSettings.authenticated(endpoints, telemetry.database(), + telemetry.username().orElseThrow()) + : GreptimeSettings.anonymous(endpoints, telemetry.database()); + ManagedApplicationConfig application = new ManagedApplicationConfig( + new MetadataDatabaseSettings(request.metadata().kind(), request.metadata().jdbcUrl(), + request.metadata().username()), telemetrySettings); + return new ManagedConfigurationBundle(application, + new ManagedSecrets(SecretValue.copyOf(request.metadata().password()), + telemetry.password().map(SecretValue::copyOf))); + } + + private static GreptimeEndpoints endpoints(TelemetryStoreConfiguration telemetry) { + return new GreptimeEndpoints(telemetry.grpcEndpoints(), telemetry.httpEndpoint()); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationProjection.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationProjection.java new file mode 100644 index 0000000000..bda9badc55 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationProjection.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.List; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigSource; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ManagementDatabaseSummary; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionalConfigurationSummary; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreSummary; + +/** Secret-free effective configuration state displayed by setup status. */ +public record SetupConfigurationProjection( + ManagementDatabaseSummary managementDatabase, + TelemetryStoreSummary telemetryStore, + OptionalConfigurationSummary optional, + List warnings) { + + public SetupConfigurationProjection { + warnings = List.copyOf(warnings); + } + + public static SetupConfigurationProjection defaults() { + return new SetupConfigurationProjection( + new ManagementDatabaseSummary(MetadataDatabaseKind.H2, false, + ConfigSource.BUILT_IN_DEFAULT, false), + new TelemetryStoreSummary(TelemetryStoreKind.GREPTIME, false, + ConfigSource.BUILT_IN_DEFAULT, false), + new OptionalConfigurationSummary(false, false, false, false, false), + List.of(SetupWarningCode.H2_NON_PRODUCTION)); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupExportRenderer.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupExportRenderer.java new file mode 100644 index 0000000000..f2326eb2d9 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupExportRenderer.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.DATASOURCE_PASSWORD; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.DATASOURCE_URL; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.DATASOURCE_USERNAME; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.GREPTIME_DATABASE; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.GREPTIME_GRPC; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.GREPTIME_HTTP; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.GREPTIME_PASSWORD; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.GREPTIME_USERNAME; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Base64; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportResponse; +import org.apache.hertzbeat.manager.setup.config.ExternalConfigExportArtifact; +import org.apache.hertzbeat.manager.setup.config.SensitiveExportContent; + +/** Renders the frozen export formats entirely in memory without writing setup secrets to disk. */ +public final class SetupExportRenderer { + + public ExternalConfigExportArtifact render(ExportRequest request, ExportResponse metadata) { + String content = switch (request.format()) { + case YAML -> yaml(request.configuration()); + case ENV -> environment(request.configuration()); + case KUBERNETES_SECRET -> kubernetesSecret(request.configuration()); + }; + byte[] bytes = content.getBytes(StandardCharsets.UTF_8); + try { + return new ExternalConfigExportArtifact(metadata.fileName(), metadata.mediaType(), + SensitiveExportContent.of(bytes)); + } finally { + Arrays.fill(bytes, (byte) 0); + } + } + + private static String yaml(ConfigurationRequest request) { + var metadata = request.managementDatabase(); + var telemetry = request.telemetryStore(); + StringBuilder output = new StringBuilder(); + yaml(output, DATASOURCE_URL, metadata.jdbcUrl()); + yaml(output, DATASOURCE_USERNAME, metadata.username()); + yaml(output, DATASOURCE_PASSWORD, metadata.password()); + yaml(output, GREPTIME_GRPC, telemetry.grpcEndpoints()); + yaml(output, GREPTIME_HTTP, telemetry.httpEndpoint()); + yaml(output, GREPTIME_DATABASE, telemetry.database()); + if (telemetry.username() != null) { + yaml(output, GREPTIME_USERNAME, telemetry.username()); + yaml(output, GREPTIME_PASSWORD, telemetry.password()); + } + return output.toString(); + } + + private static String environment(ConfigurationRequest request) { + var metadata = request.managementDatabase(); + var telemetry = request.telemetryStore(); + StringBuilder output = new StringBuilder(); + env(output, "SPRING_DATASOURCE_URL", metadata.jdbcUrl()); + env(output, "SPRING_DATASOURCE_USERNAME", metadata.username()); + env(output, "SPRING_DATASOURCE_PASSWORD", metadata.password()); + env(output, "WAREHOUSE_STORE_GREPTIME_GRPC_ENDPOINTS", telemetry.grpcEndpoints()); + env(output, "WAREHOUSE_STORE_GREPTIME_HTTP_ENDPOINT", telemetry.httpEndpoint()); + env(output, "WAREHOUSE_STORE_GREPTIME_DATABASE", telemetry.database()); + if (telemetry.username() != null) { + env(output, "WAREHOUSE_STORE_GREPTIME_USERNAME", telemetry.username()); + env(output, "WAREHOUSE_STORE_GREPTIME_PASSWORD", telemetry.password()); + } + return output.toString(); + } + + private static String kubernetesSecret(ConfigurationRequest request) { + StringBuilder output = new StringBuilder("apiVersion: v1\nkind: Secret\nmetadata:\n" + + " name: hertzbeat-setup\ntype: Opaque\ndata:\n"); + data(output, "managed-application.yml", yaml(request)); + data(output, "managed-setup.env", environment(request)); + return output.toString(); + } + + private static void yaml(StringBuilder output, String key, String value) { + output.append(key).append(": '").append(value.replace("'", "''")).append("'\n"); + } + + private static void env(StringBuilder output, String key, String value) { + output.append(key).append('=').append(environmentValue(value)).append('\n'); + } + + private static String environmentValue(String value) { + if (value.matches("[A-Za-z0-9_./:@+\\-]*")) { + return value; + } + return "'" + value.replace("'", "'\"'\"'") + "'"; + } + + private static void data(StringBuilder output, String key, String value) { + output.append(" ").append(key).append(": ").append(Base64.getEncoder() + .encodeToString(value.getBytes(StandardCharsets.UTF_8))).append('\n'); + } + +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupMutationSerializer.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupMutationSerializer.java new file mode 100644 index 0000000000..03a5c2dc0e --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupMutationSerializer.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.function.Supplier; + +/** One shared serialization boundary for durable setup mutations and their state transitions. */ +public final class SetupMutationSerializer { + + public synchronized T execute(Supplier mutation) { + return mutation.get(); + } + + public synchronized void execute(Runnable mutation) { + mutation.run(); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOperationRegistry.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOperationRegistry.java new file mode 100644 index 0000000000..5ab58fa3b6 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOperationRegistry.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.time.Clock; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.UUID; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OperationResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; + +/** Bounded, secret-free setup operation registry with one active mutation at a time. */ +public final class SetupOperationRegistry { + private static final int MAX_HISTORY = 64; + private static final long POLL_AFTER_MILLIS = 1_000; + private final Clock clock; + private final Map operations = new LinkedHashMap<>(); + private String activeOperationId; + + public SetupOperationRegistry(Clock clock) { + this.clock = clock; + } + + public synchronized String begin(SetupPhase phase) { + if (activeOperationId != null) { + throw new SetupWorkflowConflict(); + } + String id = UUID.randomUUID().toString(); + Instant now = clock.instant(); + operations.put(id, new OperationResponse(id, SetupOperationState.RUNNING, phase, + now, now, null, null, POLL_AFTER_MILLIS, false)); + activeOperationId = id; + trimHistory(); + return id; + } + + public synchronized OperationResponse finish( + String id, SetupOperationState state, SetupPhase phase, SetupErrorCode errorCode, boolean exportAvailable) { + OperationResponse current = require(id); + Instant completedAt = terminal(state) ? clock.instant() : null; + OperationResponse updated = new OperationResponse(id, state, phase, current.createdAt(), + current.startedAt(), completedAt, errorCode, + terminal(state) ? 0 : POLL_AFTER_MILLIS, exportAvailable); + operations.put(id, updated); + if (terminal(state)) { + activeOperationId = null; + } + return updated; + } + + public synchronized OperationResponse get(String id) { + return operations.get(id); + } + + private OperationResponse require(String id) { + OperationResponse operation = operations.get(id); + if (operation == null || !id.equals(activeOperationId)) { + throw new SetupWorkflowConflict(); + } + return operation; + } + + private void trimHistory() { + while (operations.size() > MAX_HISTORY) { + String first = operations.keySet().iterator().next(); + if (first.equals(activeOperationId)) { + return; + } + operations.remove(first); + } + } + + private static boolean terminal(SetupOperationState state) { + return state == SetupOperationState.SUCCEEDED || state == SetupOperationState.FAILED + || state == SetupOperationState.ROLLED_BACK; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOptionsCoordinator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOptionsCoordinator.java new file mode 100644 index 0000000000..caabf0221d --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOptionsCoordinator.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.IOException; +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiException; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.ManagedOptionalConfiguration; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.springframework.http.HttpStatus; + +/** Maps and atomically persists optional setup settings through the existing two-file transaction. */ +public final class SetupOptionsCoordinator { + private final ManagedConfigurationTransaction transaction; + + public SetupOptionsCoordinator(ManagedConfigurationTransaction transaction) { + this.transaction = transaction; + } + + public void persist(OptionsRequest request) { + ManagedOptionalConfiguration options = new ManagedOptionalConfiguration( + Optional.ofNullable(request.publicAccess()).map(value -> + new ManagedOptionalConfiguration.PublicAccessSettings( + text(value.publicBaseUrl()), text(value.serverOtlpHttpEndpoint()), + text(value.serverOtlpGrpcEndpoint()))), + Optional.ofNullable(request.retention()).map(value -> + new ManagedOptionalConfiguration.RetentionSettings( + value.metricsDays(), value.logsDays(), value.tracesDays())), + Optional.ofNullable(request.mail()).map(value -> + new ManagedOptionalConfiguration.MailSettings(value.host(), value.port(), value.security(), + text(value.username()), value.fromAddress()))); + Optional mailPassword = Optional.ofNullable(request.mail()) + .flatMap(value -> text(value.password())).map(SecretValue::of); + try { + ManagedConfigurationTransaction.Outcome outcome = transaction.applyOptions(options, mailPassword); + if (outcome == ManagedConfigurationTransaction.Outcome.RECOVERY_REQUIRED) { + throw new SetupApiException(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, HttpStatus.CONFLICT); + } + if (outcome != ManagedConfigurationTransaction.Outcome.APPLIED) { + throw writeFailure(); + } + } catch (IOException failure) { + throw writeFailure(); + } finally { + mailPassword.ifPresent(SecretValue::close); + } + } + + private static Optional text(String value) { + return value == null || value.isBlank() ? Optional.empty() : Optional.of(value); + } + + private static SetupApiException writeFailure() { + return new SetupApiException(SetupErrorCode.CONFIG_WRITE_FAILED, HttpStatus.INTERNAL_SERVER_ERROR); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidator.java new file mode 100644 index 0000000000..c129bda248 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidator.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.time.Clock; +import java.time.Duration; +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.SetupApiException; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidateRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationResponse; +import org.apache.hertzbeat.manager.setup.workflow.MetadataConfigurationValidator.Validation; +import org.springframework.http.HttpStatus; + +/** Shared browser/headless validation dispatcher with section-specific validators. */ +public final class SetupRequestValidator { + private final Clock clock; + private final MetadataConfigurationValidator metadata = new MetadataConfigurationValidator(); + private final TelemetryConfigurationValidator telemetry = new TelemetryConfigurationValidator(); + private final PublicAccessConfigurationValidator publicAccess = new PublicAccessConfigurationValidator(); + private final MailConfigurationValidator mail = new MailConfigurationValidator(); + private final MetadataConnectionProbe metadataConnection; + private final TelemetryConnectionProbe telemetryConnection; + private final MailConnectionProbe mailConnection; + + public SetupRequestValidator(Clock clock) { + this(clock, new JdbcMetadataConnectionProbe(Duration.ofSeconds(5)), + new GreptimeHttpConnectionProbe(Duration.ofSeconds(5)), + new JakartaMailConnectionProbe(Duration.ofSeconds(5))); + } + + public SetupRequestValidator(Clock clock, MetadataConnectionProbe metadataConnection, + TelemetryConnectionProbe telemetryConnection, + MailConnectionProbe mailConnection) { + this.clock = clock; + this.metadataConnection = metadataConnection; + this.telemetryConnection = telemetryConnection; + this.mailConnection = mailConnection; + } + + public ValidationResponse validate(ValidateRequest request) { + Validation structural = switch (request.section()) { + case METADATA_DATABASE -> metadata.validate(request.managementDatabase()); + case TELEMETRY_STORE -> telemetry.validate(request.telemetryStore()); + case PUBLIC_ACCESS -> publicAccess.validate(request.publicAccess()); + case MAIL -> mail.validate(request.mail()); + }; + Validation result = structural.valid() ? liveValidation(request, structural) : structural; + return new ValidationResponse(result.valid(), clock.instant(), result.errorCode(), result.warnings()); + } + + public void validate(HeadlessSetupWorkflow.Metadata configuration) { + Validation structural = metadata.validate(configuration.kind(), configuration.jdbcUrl()); + if (!structural.valid()) { + throw new SetupApiException(structural.errorCode(), HttpStatus.BAD_REQUEST); + } + requireSuccess(metadataConnection.probe(configuration)); + } + + public void validate(HeadlessSetupWorkflow.Telemetry configuration) { + Validation structural = telemetry.validate(configuration.grpcEndpoints(), configuration.httpEndpoint(), + configuration.username().isPresent(), configuration.password().isPresent()); + if (!structural.valid()) { + throw new SetupApiException(SetupErrorCode.TELEMETRY_CONNECTION_FAILED, HttpStatus.BAD_REQUEST); + } + requireSuccess(telemetryConnection.probe(configuration)); + } + + private Validation liveValidation(ValidateRequest request, Validation structural) { + Optional failure = switch (request.section()) { + case METADATA_DATABASE -> metadataConnection.probe(request.managementDatabase()); + case TELEMETRY_STORE -> telemetryConnection.probe(request.telemetryStore()); + case MAIL -> mailConnection.probe(request.mail()); + case PUBLIC_ACCESS -> Optional.empty(); + }; + return failure.map(Validation::failed).orElse(structural); + } + + private static void requireSuccess(Optional failure) { + failure.ifPresent(code -> { + throw new SetupApiException(code, HttpStatus.BAD_REQUEST); + }); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupRuntimeState.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupRuntimeState.java new file mode 100644 index 0000000000..e351fd47ea --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupRuntimeState.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.time.Clock; +import java.util.List; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionalConfigurationSummary; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupAccess; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.StatusResponse; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; + +/** Synchronized secret-free setup projection; durable stores remain the source of completion truth. */ +public final class SetupRuntimeState { + private final Clock clock; + private final ManagedConfigCapability capability; + private SetupPhase phase; + private SetupAccess access; + private boolean administratorConfigured; + private String administratorUsername; + private SetupConfigurationProjection configuration; + private String operationId; + + public SetupRuntimeState(Clock clock, ManagedConfigCapability capability, SetupPhase phase, + SetupAccess access, boolean administratorConfigured, + String administratorUsername) { + this(clock, capability, phase, access, administratorConfigured, + administratorUsername, SetupConfigurationProjection.defaults()); + } + + public SetupRuntimeState(Clock clock, ManagedConfigCapability capability, SetupPhase phase, + SetupAccess access, boolean administratorConfigured, + String administratorUsername, SetupConfigurationProjection configuration) { + this.clock = clock; + this.capability = capability; + this.phase = phase; + this.access = access; + this.administratorConfigured = administratorConfigured; + this.administratorUsername = administratorUsername; + this.configuration = configuration; + } + + public synchronized StatusResponse status() { + return new StatusResponse(phase, clock.instant(), access, capability.applyMode(), + capability.writableManagedConfig(), operationId, errorFor(phase), + configuration.managementDatabase(), configuration.telemetryStore(), + administratorConfigured, configuration.optional(), configuration.warnings()); + } + + public synchronized SetupPhase phase() { + return phase; + } + + public synchronized void ensurePhase(SetupPhase expected) { + requirePhase(expected); + } + + public synchronized void unlocked() { + if (access == SetupAccess.UNLOCKED) { + return; + } + if (access != SetupAccess.LOCKED) { + throw new SetupWorkflowConflict(); + } + access = SetupAccess.UNLOCKED; + } + + /** Revokes remote mutation access when a fresh owner proof replaces an expired session. */ + public synchronized void locked() { + if (access == SetupAccess.LOCKED) { + return; + } + if (access != SetupAccess.UNLOCKED) { + throw new SetupWorkflowConflict(); + } + access = SetupAccess.LOCKED; + } + + public synchronized void configurationApplied(String id, SetupPhase next) { + requirePhase(SetupPhase.CONFIGURATION_REQUIRED); + operationId = id; + phase = next; + } + + public synchronized void administratorCreated(String username) { + requirePhase(SetupPhase.ADMINISTRATOR_REQUIRED); + administratorConfigured = true; + administratorUsername = username; + phase = SetupPhase.OPTIONAL_CONFIGURATION; + } + + public synchronized void optionsConfigured(OptionalConfigurationSummary configured, + List warnings) { + requirePhase(SetupPhase.OPTIONAL_CONFIGURATION); + configuration = new SetupConfigurationProjection(configuration.managementDatabase(), + configuration.telemetryStore(), configured, warnings); + } + + public synchronized List pendingWarnings() { + return configuration.warnings(); + } + + public synchronized MetadataDatabaseKind managementDatabaseKind() { + return configuration.managementDatabase().kind(); + } + + public synchronized String administratorUsername() { + return administratorUsername; + } + + public synchronized void complete() { + requirePhase(SetupPhase.OPTIONAL_CONFIGURATION); + phase = SetupPhase.COMPLETE; + operationId = null; + } + + private void requirePhase(SetupPhase expected) { + if (phase != expected) { + throw new SetupWorkflowConflict(); + } + } + + private static SetupErrorCode errorFor(SetupPhase phase) { + return phase == SetupPhase.RECOVERY_REQUIRED ? SetupErrorCode.CONFIG_RECOVERY_REQUIRED : null; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupWarningPolicy.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupWarningPolicy.java new file mode 100644 index 0000000000..9ba81424b0 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupWarningPolicy.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; + +/** Single warning policy shared by live setup and restart status projection. */ +public final class SetupWarningPolicy { + public static final SetupWarningPolicy INSTANCE = new SetupWarningPolicy(); + + private SetupWarningPolicy() { + } + + public List evaluate(MetadataDatabaseKind kind, OptionsRequest options) { + String publicUrl = options.publicAccess() == null ? null : options.publicAccess().publicBaseUrl(); + MailSecurity mailSecurity = options.mail() == null ? null : options.mail().security(); + return evaluate(kind, publicUrl, mailSecurity); + } + + public List evaluate( + MetadataDatabaseKind kind, String publicUrl, MailSecurity mailSecurity) { + List warnings = new ArrayList<>(); + if (kind == MetadataDatabaseKind.H2) { + warnings.add(SetupWarningCode.H2_NON_PRODUCTION); + } + if (publicUrl != null && publicUrl.toLowerCase(Locale.ROOT).startsWith("http://")) { + warnings.add(SetupWarningCode.PUBLIC_ADDRESS_PLAINTEXT); + } + if (mailSecurity == MailSecurity.NONE) { + warnings.add(SetupWarningCode.MAIL_SECURITY_NONE); + } + return List.copyOf(warnings); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupWorkflowConflict.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupWorkflowConflict.java new file mode 100644 index 0000000000..bbccf8ddb2 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupWorkflowConflict.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +/** Safe state-machine conflict without request or secret values. */ +public final class SetupWorkflowConflict extends IllegalStateException { + public SetupWorkflowConflict() { + super("Setup workflow state conflict"); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TelemetryConfigurationValidator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TelemetryConfigurationValidator.java new file mode 100644 index 0000000000..645934c6e7 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TelemetryConfigurationValidator.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.net.URI; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreConfiguration; +import org.apache.hertzbeat.manager.setup.workflow.MetadataConfigurationValidator.Validation; + +/** Validates the supported Greptime endpoint shapes without retaining credentials. */ +final class TelemetryConfigurationValidator { + Validation validate(TelemetryStoreConfiguration configuration) { + return validate(configuration.grpcEndpoints(), configuration.httpEndpoint(), + hasText(configuration.username()), hasText(configuration.password())); + } + + Validation validate(String grpcEndpoints, String httpEndpoint, + boolean usernamePresent, boolean passwordPresent) { + if (!hasHostAndPort(grpcEndpoints) || !isHttpUri(httpEndpoint) + || usernamePresent != passwordPresent) { + return Validation.failed(SetupErrorCode.TELEMETRY_CONNECTION_FAILED); + } + return Validation.success(); + } + + private static boolean hasText(String value) { + return value != null && !value.isBlank(); + } + + private static boolean hasHostAndPort(String value) { + int separator = value == null ? -1 : value.lastIndexOf(':'); + if (separator < 1 || separator == value.length() - 1) { + return false; + } + try { + int port = Integer.parseInt(value.substring(separator + 1)); + return port > 0 && port <= 65_535; + } catch (NumberFormatException failure) { + return false; + } + } + + private static boolean isHttpUri(String value) { + try { + URI uri = URI.create(value); + return uri.getHost() != null && ("http".equalsIgnoreCase(uri.getScheme()) + || "https".equalsIgnoreCase(uri.getScheme())); + } catch (IllegalArgumentException failure) { + return false; + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TelemetryConnectionProbe.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TelemetryConnectionProbe.java new file mode 100644 index 0000000000..0ae43b83de --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TelemetryConnectionProbe.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreConfiguration; +import org.apache.hertzbeat.manager.setup.config.SecretValue; + +/** Injectable live Greptime connectivity and authentication boundary. */ +@FunctionalInterface +public interface TelemetryConnectionProbe { + Optional probe(Request configuration); + + default Optional probe(TelemetryStoreConfiguration configuration) { + Optional password = configuration.password() == null + ? Optional.empty() : Optional.of(SecretValue.of(configuration.password())); + try (Request request = new Request(configuration.kind(), configuration.grpcEndpoints(), + configuration.httpEndpoint(), configuration.database(), + Optional.ofNullable(configuration.username()), password)) { + return probe(request); + } + } + + default Optional probe(HeadlessSetupWorkflow.Telemetry configuration) { + try (Request request = new Request(TelemetryStoreKind.GREPTIME, + configuration.grpcEndpoints(), configuration.httpEndpoint(), configuration.database(), + configuration.username(), configuration.password().map(SecretValue::copyOf))) { + return probe(request); + } + } + + /** Probe-scoped clearable credentials; the probe must not retain this request after return. */ + record Request(TelemetryStoreKind kind, String grpcEndpoints, String httpEndpoint, String database, + Optional username, Optional password) implements AutoCloseable { + public Request { + Objects.requireNonNull(username, "username"); + Objects.requireNonNull(password, "password"); + } + + @Override + public void close() { + password.ifPresent(SecretValue::close); + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfigurationTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfigurationTest.java new file mode 100644 index 0000000000..15ee21d563 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfigurationTest.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +class SetupApiConfigurationTest { + + @Test + void defaultAndBlankBindAddressesMustRequireRemoteUnlock() { + assertThat(SetupApiConfiguration.bindAddress(null).isAnyLocalAddress()).isTrue(); + assertThat(SetupApiConfiguration.bindAddress(" ").isAnyLocalAddress()).isTrue(); + } + + @Test + void explicitLoopbackAddressMustRemainLocal() { + assertThat(SetupApiConfiguration.bindAddress("127.0.0.1").isLoopbackAddress()).isTrue(); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java index 1f364b6aeb..ca48e9b2bc 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java @@ -79,14 +79,14 @@ class SetupApiContractTest { assertWireValues(MailSecurity.values(), "none", "starttls", "tls"); assertWireValues(SetupApiContract.ExportFormat.values(), "yaml", "env", "kubernetes_secret"); assertWireValues(SetupApiContract.SetupWarningCode.values(), "external_apply_required", "restart_required", - "public_address_plaintext", "mail_security_none"); + "public_address_plaintext", "mail_security_none", "h2_non_production"); } @Test void freezesSafeStatusAndMutationShapes() { assertComponents(SetupApiContract.StatusResponse.class, "phase", "observedAt", "access", "applyMode", "writableManagedConfig", "operationId", "errorCode", "managementDatabase", "telemetryStore", - "administratorConfigured", "optional"); + "administratorConfigured", "optional", "pendingWarnings"); assertComponents(SetupApiContract.ManagementDatabaseSummary.class, "kind", "configured", "source", "restartRequired"); assertComponents(SetupApiContract.TelemetryStoreSummary.class, "kind", "configured", "source", diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupControllerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupControllerTest.java new file mode 100644 index 0000000000..9295207f89 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupControllerTest.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.asyncDispatch; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.request; + +import java.time.Instant; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigSource; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ManagementDatabaseSummary; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionalConfigurationSummary; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupAccess; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.StatusResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreSummary; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.UnlockResponse; +import org.apache.hertzbeat.manager.setup.security.SetupUnlockRejected; +import org.apache.hertzbeat.manager.setup.security.SetupHttpUnlockService; +import org.apache.hertzbeat.manager.setup.runtime.SetupResponseTransition; +import org.apache.hertzbeat.manager.setup.workflow.SetupExportRenderer; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; + +class SetupControllerTest { + + private final SetupWorkflow workflow = mock(SetupWorkflow.class); + private MockMvc mvc; + + @BeforeEach + void setUp() { + mvc = MockMvcBuilders.standaloneSetup(new SetupController(workflow, + mock(SetupHttpUnlockService.class), mock(SetupResponseTransition.class), + new SetupExportRenderer())) + .setControllerAdvice(new SetupExceptionHandler()).build(); + } + + @Test + void exposesSafeNoStoreStatus() throws Exception { + when(workflow.status()).thenReturn(new StatusResponse( + SetupPhase.CONFIGURATION_REQUIRED, Instant.parse("2026-08-08T00:00:00Z"), + SetupAccess.LOCAL, ApplyMode.MANAGED_WRITE, true, null, null, + new ManagementDatabaseSummary(MetadataDatabaseKind.H2, false, + ConfigSource.BUILT_IN_DEFAULT, false), + new TelemetryStoreSummary(TelemetryStoreKind.GREPTIME, false, + ConfigSource.BUILT_IN_DEFAULT, false), + false, new OptionalConfigurationSummary(false, false, false, false, false))); + + mvc.perform(get(SetupApiContract.STATUS_PATH)) + .andExpect(status().isOk()) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.phase").value("configuration_required")) + .andExpect(content().string(org.hamcrest.Matchers.not(org.hamcrest.Matchers.containsString("jdbc")))); + } + + @Test + void routesEveryFrozenMutationAndOperationPath() throws Exception { + when(workflow.unlock(any())).thenReturn(new UnlockResponse( + SetupAccess.UNLOCKED, Instant.parse("2026-08-08T00:15:00Z"))); + + mvc.perform(post(SetupApiContract.UNLOCK_PATH).contentType(MediaType.APPLICATION_JSON) + .content("{\"code\":\"one-time-proof\"}")) + .andExpect(status().isOk()).andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.access").value("unlocked")); + + mvc.perform(get(SetupApiContract.OPERATION_PATH, "missing")) + .andExpect(status().isNotFound()).andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.errorCode").value("operation_not_found")); + } + + @Test + void mapsUnlockRejectionsWithoutExceptionDetails() throws Exception { + when(workflow.unlock(any())).thenThrow(new SetupUnlockRejected(SetupUnlockRejected.Reason.RATE_LIMITED)); + + mvc.perform(post(SetupApiContract.UNLOCK_PATH).contentType(MediaType.APPLICATION_JSON) + .content("{\"code\":\"invalid-proof\"}")) + .andExpect(status().isTooManyRequests()) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.errorCode").value("setup_rate_limited")) + .andExpect(content().string(org.hamcrest.Matchers.not(org.hamcrest.Matchers.containsString("exception")))) + .andExpect(content().string(org.hamcrest.Matchers.not(org.hamcrest.Matchers.containsString("invalid-proof")))); + } + + @Test + void mapsMalformedInputToStableSafeError() throws Exception { + mvc.perform(post(SetupApiContract.UNLOCK_PATH).contentType(MediaType.APPLICATION_JSON).content("{}")) + .andExpect(status().isBadRequest()) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.errorCode").value("setup_code_invalid")); + } + + @Test + void exportIsActualNoStoreAttachmentRatherThanMetadataJson() throws Exception { + when(workflow.prepareExport(any())).thenReturn( + new ExportResponse("hertzbeat-setup.env", "text/plain")); + String request = """ + {"format":"env","configuration":{"expectedPhase":"configuration_required", + "applyMode":"external_apply","managementDatabase":{"kind":"h2", + "jdbcUrl":"jdbc:h2:./data/hertzbeat","username":"sa","password":"database-secret"}, + "telemetryStore":{"kind":"greptime","grpcEndpoints":"localhost:4001", + "httpEndpoint":"http://localhost:4000","database":"public"}}} + """; + + var pending = mvc.perform(post(SetupApiContract.EXPORT_PATH) + .contentType(MediaType.APPLICATION_JSON).content(request)) + .andExpect(request().asyncStarted()) + .andReturn(); + mvc.perform(asyncDispatch(pending)) + .andExpect(status().isOk()) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(header().string("Content-Disposition", "attachment; filename=\"hertzbeat-setup.env\"")) + .andExpect(content().contentTypeCompatibleWith(MediaType.TEXT_PLAIN)) + .andExpect(content().string(org.hamcrest.Matchers.containsString( + "SPRING_DATASOURCE_PASSWORD=database-secret"))); + } + + @Test + void unexpectedSetupFailureUsesStableNoStoreEnvelope() throws Exception { + when(workflow.status()).thenThrow(new IllegalStateException("database-secret")); + + mvc.perform(get(SetupApiContract.STATUS_PATH)) + .andExpect(status().isInternalServerError()) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.errorCode").value("config_write_failed")) + .andExpect(content().string(org.hamcrest.Matchers.not( + org.hamcrest.Matchers.containsString("database-secret")))); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupRuntimeStateFactoryTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupRuntimeStateFactoryTest.java new file mode 100644 index 0000000000..368cb8f76a --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupRuntimeStateFactoryTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.net.InetAddress; +import java.nio.file.Path; +import java.util.Optional; +import org.apache.hertzbeat.common.runtime.BusinessRuntimeGate; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.mock.env.MockEnvironment; + +class SetupRuntimeStateFactoryTest { + @TempDir + private Path root; + + @Test + void recoveryRuntimePublishesTheStableManagedRecoveryReason() throws Exception { + BusinessRuntimeGate gate = mock(BusinessRuntimeGate.class); + when(gate.mode()).thenReturn(RuntimeMode.RECOVERY); + + var state = new SetupRuntimeStateFactory().create(new MockEnvironment(), root, + InetAddress.getLoopbackAddress(), gate, mock(ManagedConfigCapability.class), + Optional.empty(), Optional.empty()); + + assertThat(state.status().phase()).isEqualTo(SetupPhase.RECOVERY_REQUIRED); + assertThat(state.status().errorCode()).isEqualTo(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupStatusProjectionFactoryTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupStatusProjectionFactoryTest.java new file mode 100644 index 0000000000..4133e415a5 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupStatusProjectionFactoryTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Map; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigSource; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; +import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector; +import org.junit.jupiter.api.Test; +import org.springframework.core.env.MapPropertySource; +import org.springframework.core.env.StandardEnvironment; + +class SetupStatusProjectionFactoryTest { + @Test + void restartProjectionUsesEffectiveSourceAndRehydratesSafeManagedOptions() { + StandardEnvironment environment = new StandardEnvironment(); + environment.getPropertySources().replace( + StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME, + new MapPropertySource(StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME, + Map.of("spring.jpa.database", "POSTGRESQL"))); + environment.getPropertySources().addLast(new MapPropertySource( + ManagedActiveConfigurationInspector.MANAGED_APPLICATION_SOURCE, + Map.of("spring.jpa.database", "H2", + "warehouse.store.greptime.enabled", "true", + "hertzbeat.setup.public-base-url", "http://localhost:1157", + "hertzbeat.setup.retention.metrics-days", "30", + "spring.mail.host", "mail.example.test", + "hertzbeat.setup.mail.security", "NONE"))); + var inspection = new ManagedActiveConfigurationInspector.Inspection( + ManagedActiveConfigurationInspector.State.LOADABLE, Map.of(), Map.of()); + + var projection = new SetupStatusProjectionFactory().create(environment, inspection); + + assertThat(projection.managementDatabase().kind()).isEqualTo(MetadataDatabaseKind.POSTGRESQL); + assertThat(projection.managementDatabase().source()).isEqualTo(ConfigSource.SYSTEM_PROPERTY); + assertThat(projection.optional().publicAccessConfigured()).isTrue(); + assertThat(projection.optional().retentionConfigured()).isTrue(); + assertThat(projection.optional().mailConfigured()).isTrue(); + assertThat(projection.warnings()).containsExactlyInAnyOrder( + SetupWarningCode.PUBLIC_ADDRESS_PLAINTEXT, SetupWarningCode.MAIL_SECURITY_NONE); + } + + @Test + void unknownExternalMailSecurityDoesNotBreakRestartProjection() { + StandardEnvironment environment = new StandardEnvironment(); + environment.getPropertySources().replace( + StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME, + new MapPropertySource(StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME, Map.of( + "spring.jpa.database", "MYSQL", + "hertzbeat.setup.mail.security", "legacy-value"))); + var inspection = new ManagedActiveConfigurationInspector.Inspection( + ManagedActiveConfigurationInspector.State.ABSENT, Map.of(), Map.of()); + + var projection = new SetupStatusProjectionFactory().create(environment, inspection); + + assertThat(projection.warnings()).doesNotContain(SetupWarningCode.MAIL_SECURITY_NONE); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java index 15e3038a6d..fb5dac4b94 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java @@ -19,6 +19,9 @@ package org.apache.hertzbeat.manager.setup.config; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.io.IOException; import java.nio.channels.FileChannel; @@ -167,6 +170,24 @@ class ManagedConfigurationTransactionTest { assertEquals(0, holder.exitValue()); } + @Test + void recoveryClosesEveryDecodedSecretSnapshotItOwns() throws Exception { + ManagedApplicationConfigStore applications = mock(ManagedApplicationConfigStore.class); + ManagedSecretStore secretStore = mock(ManagedSecretStore.class); + ManagedSecrets decoded = secrets("owned"); + when(applications.readActive()).thenReturn(CandidateRead.valid(configuration("owned"), "generation")); + when(applications.readCandidate()).thenReturn(CandidateRead.missing()); + when(applications.readLastKnownGood()).thenReturn(CandidateRead.missing()); + when(secretStore.readActive()).thenReturn(CandidateRead.valid(decoded, "generation")); + when(secretStore.readCandidate()).thenReturn(CandidateRead.missing()); + when(secretStore.readLastKnownGood()).thenReturn(CandidateRead.missing()); + + assertEquals(ManagedConfigurationTransaction.Outcome.APPLIED, + new ManagedConfigurationTransaction(applications, secretStore, installationRoot).recover()); + + assertThat(decoded.metadataDatabasePassword().copy()).containsOnly('\0'); + } + private static void waitForFile(Path ready) throws Exception { long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); while (!Files.exists(ready) && System.nanoTime() < deadline) { diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionalConfigurationPersistenceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionalConfigurationPersistenceTest.java new file mode 100644 index 0000000000..e6f487197a --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionalConfigurationPersistenceTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class ManagedOptionalConfigurationPersistenceTest { + @TempDir + private java.nio.file.Path root; + + @Test + void optionsUpdatePreservesRequiredSettingsAndStoresMailPasswordOnlyInSecrets() throws Exception { + ManagedConfigurationTransaction transaction = new ManagedConfigurationTransaction(root); + assertThat(transaction.apply(required())).isEqualTo(ManagedConfigurationTransaction.Outcome.APPLIED); + ManagedOptionalConfiguration options = new ManagedOptionalConfiguration( + Optional.of(new ManagedOptionalConfiguration.PublicAccessSettings( + Optional.of("https://hertzbeat.example"), + Optional.of("https://hertzbeat.example/otlp"), Optional.of("hertzbeat.example:4317"))), + Optional.of(new ManagedOptionalConfiguration.RetentionSettings(30, 14, 7)), + Optional.of(new ManagedOptionalConfiguration.MailSettings( + "smtp.example", 465, MailSecurity.TLS, Optional.of("mailer"), "alerts@example.test"))); + + assertThat(transaction.applyOptions(options, Optional.of(SecretValue.of("mail-secret")))) + .isEqualTo(ManagedConfigurationTransaction.Outcome.APPLIED); + + ManagedApplicationConfig application = new FileManagedApplicationConfigStore(root) + .readActive().value().orElseThrow(); + ManagedSecrets secrets = new FileManagedSecretStore(root).readActive().value().orElseThrow(); + assertThat(application.metadataDatabase()).isEqualTo(required().application().metadataDatabase()); + assertThat(application.optional()).isEqualTo(options); + assertThat(secrets.mailPassword()).get().isEqualTo(SecretValue.of("mail-secret")); + assertThat(ApplicationConfigDocumentCodec.springProperties(application).toString()) + .doesNotContain("mail-secret"); + } + + private static ManagedConfigurationBundle required() { + ManagedApplicationConfig application = new ManagedApplicationConfig( + new MetadataDatabaseSettings(MetadataDatabaseKind.H2, "jdbc:h2:./data/hertzbeat", "sa"), + GreptimeSettings.anonymous( + new GreptimeEndpoints("localhost:4001", "http://localhost:4000"), "public")); + return new ManagedConfigurationBundle(application, + ManagedSecrets.withoutTelemetryPassword(SecretValue.of("database-secret"))); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/installation/InstallationConvergenceServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/installation/InstallationConvergenceServiceTest.java new file mode 100644 index 0000000000..b119ce3661 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/installation/InstallationConvergenceServiceTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.installation; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.SecureRandom; +import java.util.Optional; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class InstallationConvergenceServiceTest { + @TempDir + private Path temporaryDirectory; + + @Test + void matchingDatabaseAndLocalFingerprintsAreRequiredForFullRuntime() throws Exception { + Path fingerprintPath = temporaryDirectory.resolve("fingerprint"); + InstallationFingerprint fingerprint = new LocalInstallationFingerprintStore( + fingerprintPath, new SecureRandom()).create(); + InstallationRecordRepository records = mock(InstallationRecordRepository.class); + when(records.findById(InstallationRecord.SINGLETON_ID)) + .thenReturn(Optional.of(new InstallationRecord(fingerprint.value()))); + + assertThat(new InstallationConvergenceService(records, fingerprintPath).classify()) + .isEqualTo(InstallationMode.FULL); + + when(records.findById(InstallationRecord.SINGLETON_ID)) + .thenReturn(Optional.of(new InstallationRecord("f".repeat(64)))); + assertThat(new InstallationConvergenceService(records, fingerprintPath).classify()) + .isEqualTo(InstallationMode.RECOVERY); + } + + @Test + void fingerprintWrittenBeforeDatabaseRecordRemainsGatedAndCanConverge() throws Exception { + Path fingerprintPath = temporaryDirectory.resolve("fingerprint"); + new LocalInstallationFingerprintStore(fingerprintPath, new SecureRandom()).create(); + InstallationRecordRepository records = mock(InstallationRecordRepository.class); + when(records.findById(InstallationRecord.SINGLETON_ID)).thenReturn(Optional.empty()); + + assertThat(new InstallationConvergenceService(records, fingerprintPath).classify()) + .isEqualTo(InstallationMode.UPGRADE); + } + + @Test + void malformedLocalFingerprintFailsClosed() throws Exception { + Path fingerprintPath = temporaryDirectory.resolve("fingerprint"); + Files.writeString(fingerprintPath, "not-a-fingerprint"); + InstallationRecordRepository records = mock(InstallationRecordRepository.class); + + assertThat(new InstallationConvergenceService(records, fingerprintPath).classify()) + .isEqualTo(InstallationMode.RECOVERY); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupResponseTransitionFilterTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupResponseTransitionFilterTest.java new file mode 100644 index 0000000000..0eb704d186 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupResponseTransitionFilterTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.runtime; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; + +class SetupResponseTransitionFilterTest { + + @Test + void configurationTransitionRunsOnlyAfterResponseCommitAndApplicationReadiness() throws Exception { + SetupRuntimeTransition transition = mock(SetupRuntimeTransition.class); + List tasks = new ArrayList<>(); + SetupRuntimeTransitionScheduler scheduler = new SetupRuntimeTransitionScheduler(transition, tasks::add); + SetupResponseTransition marker = new SetupResponseTransition(); + SetupResponseTransitionFilter filter = new SetupResponseTransitionFilter(scheduler, marker); + MockHttpServletRequest request = new MockHttpServletRequest("POST", "/api/setup/configuration"); + MockHttpServletResponse response = new MockHttpServletResponse(); + + filter.doFilter(request, response, (servletRequest, servletResponse) -> { + marker.arm(servletRequest); + servletResponse.getWriter().write("accepted"); + }); + + assertThat(response.isCommitted()).isTrue(); + assertThat(tasks).isEmpty(); + verify(transition, never()).configurationApplied(); + + scheduler.onApplicationReady(mock(ApplicationReadyEvent.class)); + assertThat(tasks).hasSize(1); + tasks.removeFirst().run(); + verify(transition).configurationApplied(); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransitionSchedulerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransitionSchedulerTest.java new file mode 100644 index 0000000000..49df5325f1 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransitionSchedulerTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.runtime; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; +import org.springframework.boot.context.event.ApplicationReadyEvent; + +class SetupRuntimeTransitionSchedulerTest { + @Test + void runningConfigurationCoalescesDuplicatesAndQueuesOneHigherPriorityCompletion() { + SetupRuntimeTransition transition = mock(SetupRuntimeTransition.class); + List tasks = new ArrayList<>(); + SetupRuntimeTransitionScheduler scheduler = new SetupRuntimeTransitionScheduler(transition, tasks::add); + scheduler.onApplicationReady(mock(ApplicationReadyEvent.class)); + + scheduler.configurationApplied(); + scheduler.configurationApplied(); + scheduler.installationCompleted(); + scheduler.installationCompleted(); + + assertThat(tasks).hasSize(1); + tasks.removeFirst().run(); + verify(transition).configurationApplied(); + assertThat(tasks).hasSize(1); + tasks.removeFirst().run(); + verify(transition).completeSetup(); + assertThat(tasks).isEmpty(); + } + + @Test + void completionSupersedesConfigurationBeforeReadiness() { + SetupRuntimeTransition transition = mock(SetupRuntimeTransition.class); + List tasks = new ArrayList<>(); + SetupRuntimeTransitionScheduler scheduler = new SetupRuntimeTransitionScheduler(transition, tasks::add); + scheduler.configurationApplied(); + scheduler.installationCompleted(); + + scheduler.onApplicationReady(mock(ApplicationReadyEvent.class)); + tasks.removeFirst().run(); + + verify(transition).completeSetup(); + verify(transition, times(0)).configurationApplied(); + assertThat(tasks).isEmpty(); + } + + @Test + void transitionCanCloseSchedulerFromItsExecutorWithoutInterruptingOrDispatchingPendingWork() + throws InterruptedException { + ExecutorService executor = Executors.newSingleThreadExecutor(); + AtomicReference schedulerReference = new AtomicReference<>(); + CountDownLatch transitionStarted = new CountDownLatch(1); + CountDownLatch allowClose = new CountDownLatch(1); + CountDownLatch transitionFinished = new CountDownLatch(1); + AtomicBoolean interrupted = new AtomicBoolean(); + AtomicReference failure = new AtomicReference<>(); + AtomicInteger configurationCalls = new AtomicInteger(); + AtomicInteger completionCalls = new AtomicInteger(); + SetupRuntimeTransition transition = new SetupRuntimeTransition() { + @Override + public void configurationApplied() { + configurationCalls.incrementAndGet(); + transitionStarted.countDown(); + try { + allowClose.await(); + schedulerReference.get().close(); + interrupted.set(Thread.currentThread().isInterrupted()); + } catch (Throwable error) { + failure.set(error); + } finally { + transitionFinished.countDown(); + } + } + + @Override + public void completeSetup() { + completionCalls.incrementAndGet(); + } + }; + SetupRuntimeTransitionScheduler scheduler = new SetupRuntimeTransitionScheduler(transition, executor); + schedulerReference.set(scheduler); + scheduler.onApplicationReady(mock(ApplicationReadyEvent.class)); + + scheduler.configurationApplied(); + assertThat(transitionStarted.await(5, TimeUnit.SECONDS)).isTrue(); + scheduler.installationCompleted(); + allowClose.countDown(); + + assertThat(transitionFinished.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(executor.awaitTermination(5, TimeUnit.SECONDS)).isTrue(); + assertThat(failure.get()).isNull(); + assertThat(interrupted).isFalse(); + assertThat(configurationCalls).hasValue(1); + assertThat(completionCalls).hasValue(0); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/RemoteSetupUnlockTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/RemoteSetupUnlockTest.java index 51898d1ff3..ebc0d80996 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/RemoteSetupUnlockTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/RemoteSetupUnlockTest.java @@ -34,6 +34,7 @@ import java.security.SecureRandom; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.time.ZoneId; import java.time.ZoneOffset; import java.util.Arrays; import org.junit.jupiter.api.Test; @@ -138,6 +139,39 @@ class RemoteSetupUnlockTest { assertFalse(unlock.permits(oldSession.token())); } + @Test + void ensureOpenPreservesAnActiveSession() throws Exception { + Path codeFile = temporaryDirectory.resolve("unlock"); + MutableClock clock = new MutableClock(Instant.parse("2026-08-08T00:00:00Z")); + RemoteSetupUnlock unlock = new RemoteSetupUnlock(codeFile, clock, deterministicRandom()); + unlock.ensureOpen(); + SetupAccessSession session = unlock.redeem( + "198.51.100.4", new SetupUnlockCode(Files.readString(codeFile).toCharArray())); + + unlock.ensureOpen(); + + assertTrue(unlock.permits(session.token())); + assertFalse(Files.exists(codeFile)); + } + + @Test + void ensureOpenRenewsProofAfterExpiry() throws Exception { + Path codeFile = temporaryDirectory.resolve("unlock"); + MutableClock clock = new MutableClock(Instant.parse("2026-08-08T00:00:00Z")); + RemoteSetupUnlock unlock = new RemoteSetupUnlock(codeFile, clock, deterministicRandom()); + unlock.ensureOpen(); + String expiredCode = Files.readString(codeFile); + SetupAccessSession expiredSession = unlock.redeem( + "198.51.100.4", new SetupUnlockCode(expiredCode.toCharArray())); + + clock.advance(Duration.ofMinutes(16)); + unlock.ensureOpen(); + + assertFalse(unlock.permits(expiredSession.token())); + assertTrue(Files.exists(codeFile)); + assertNotEquals(expiredCode, Files.readString(codeFile)); + } + @Test void failedProofRemovalDoesNotPublishUnreachableSession() throws Exception { Path codeFile = temporaryDirectory.resolve("unlock"); @@ -173,4 +207,31 @@ class RemoteSetupUnlockTest { } }; } + + private static final class MutableClock extends Clock { + private Instant instant; + + private MutableClock(Instant instant) { + this.instant = instant; + } + + void advance(Duration duration) { + instant = instant.plus(duration); + } + + @Override + public ZoneId getZone() { + return ZoneOffset.UTC; + } + + @Override + public Clock withZone(ZoneId zone) { + return this; + } + + @Override + public Instant instant() { + return instant; + } + } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SetupHttpUnlockServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SetupHttpUnlockServiceTest.java new file mode 100644 index 0000000000..650aaf9215 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SetupHttpUnlockServiceTest.java @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.security; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +import java.net.InetAddress; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermission; +import java.security.SecureRandom; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupAccess; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.UnlockRequest; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; +import org.apache.hertzbeat.manager.setup.workflow.SetupRuntimeState; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; + +class SetupHttpUnlockServiceTest { + @TempDir + private Path temporaryDirectory; + + @Test + void completedSetupMustNotPublishAnotherRemoteUnlockCode() throws Exception { + Path codeFile = temporaryDirectory.resolve("unlock-code"); + SetupRuntimeState state = new SetupRuntimeState(Clock.systemUTC(), mock(ManagedConfigCapability.class), + SetupPhase.COMPLETE, SetupAccess.LOCKED, true, "operator"); + + try (SetupHttpUnlockService service = new SetupHttpUnlockService( + new RemoteSetupUnlock(codeFile, Clock.systemUTC(), new SecureRandom()), + InetAddress.getByName("0.0.0.0"), state, Clock.systemUTC())) { + assertThat(service.requiresUnlock()).isFalse(); + assertThat(Files.exists(codeFile)).isFalse(); + } + } + + @Test + void ordinaryLoopbackSetupDoesNotPublishUnusedUnlockProof() throws Exception { + Path codeFile = temporaryDirectory.resolve("local-unlock-code"); + SetupRuntimeState state = new SetupRuntimeState(Clock.systemUTC(), mock(ManagedConfigCapability.class), + SetupPhase.CONFIGURATION_REQUIRED, SetupAccess.LOCAL, false, null); + try (SetupHttpUnlockService service = new SetupHttpUnlockService( + new RemoteSetupUnlock(codeFile, Clock.systemUTC(), new SecureRandom()), + InetAddress.getLoopbackAddress(), state, Clock.systemUTC())) { + MockHttpServletRequest direct = new MockHttpServletRequest("GET", "/api/setup/status"); + assertThat(service.requiresUnlock(direct)).isFalse(); + assertThat(Files.exists(codeFile)).isFalse(); + } + } + + @Test + void untrustedForwardingRequiresUnlockAndCannotGrantSecureCookieOrPublicPolling() throws Exception { + Path codeFile = temporaryDirectory.resolve("unlock-code"); + SetupRuntimeState state = new SetupRuntimeState(Clock.systemUTC(), mock(ManagedConfigCapability.class), + SetupPhase.CONFIGURATION_REQUIRED, SetupAccess.LOCAL, false, null); + try (SetupHttpUnlockService service = new SetupHttpUnlockService( + new RemoteSetupUnlock(codeFile, Clock.systemUTC(), new SecureRandom()), + InetAddress.getLoopbackAddress(), state, Clock.systemUTC())) { + MockHttpServletRequest direct = new MockHttpServletRequest("GET", "/api/setup/status"); + assertThat(service.requiresUnlock(direct)).isFalse(); + MockHttpServletRequest forwarded = new MockHttpServletRequest( + "GET", "/api/setup/operations/operation-1"); + forwarded.setServletPath("/api/setup/operations/operation-1"); + forwarded.addHeader("X-Forwarded-Proto", "https"); + forwarded.setSecure(true); + assertThat(service.requiresUnlock(forwarded)).isTrue(); + assertThat(service.secureCookie(forwarded)).isFalse(); + + MockHttpServletResponse response = new MockHttpServletResponse(); + new SetupWriteAccessFilter(service, Clock.systemUTC()).doFilter( + forwarded, response, (request, target) -> request.setAttribute("called", true)); + assertThat(response.getStatus()).isEqualTo(403); + assertThat(forwarded.getAttribute("called")).isNull(); + } + } + + @Test + void forwardedLoopbackCanRedeemOwnerProofAndPollWithConservativeCookie() throws Exception { + Path codeFile = temporaryDirectory.resolve("unlock-code"); + SetupRuntimeState state = new SetupRuntimeState(Clock.systemUTC(), mock(ManagedConfigCapability.class), + SetupPhase.CONFIGURATION_REQUIRED, SetupAccess.LOCAL, false, null); + try (SetupHttpUnlockService service = new SetupHttpUnlockService( + new RemoteSetupUnlock(codeFile, Clock.systemUTC(), new SecureRandom()), + InetAddress.getLoopbackAddress(), state, Clock.systemUTC())) { + MockHttpServletRequest forwarded = new MockHttpServletRequest( + "POST", "/api/setup/unlock"); + forwarded.setRemoteAddr("127.0.0.1"); + forwarded.setSecure(true); + forwarded.addHeader("Forwarded", "for=203.0.113.4;proto=https"); + assertThat(service.requiresUnlock(forwarded)).isTrue(); + assertThat(Files.getPosixFilePermissions(codeFile)).containsExactlyInAnyOrder( + PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE); + String code = Files.readString(codeFile); + + var exchange = service.redeem(new UnlockRequest(code), forwarded); + + assertThat(exchange.cookie().isHttpOnly()).isTrue(); + assertThat(exchange.cookie().getSameSite()).isEqualTo("Strict"); + assertThat(exchange.cookie().isSecure()).isFalse(); + assertThat(state.status().access()).isEqualTo(SetupAccess.LOCAL); + + MockHttpServletRequest poll = new MockHttpServletRequest( + "GET", "/api/setup/operations/operation-1"); + poll.setServletPath("/api/setup/operations/operation-1"); + poll.addHeader("X-Forwarded-Proto", "https"); + poll.setSecure(true); + poll.setCookies(new jakarta.servlet.http.Cookie( + SetupAccessCookie.NAME, exchange.cookie().getValue())); + MockHttpServletResponse response = new MockHttpServletResponse(); + new SetupWriteAccessFilter(service, Clock.systemUTC()).doFilter( + poll, response, (request, target) -> request.setAttribute("called", true)); + + assertThat(poll.getAttribute("called")).isEqualTo(true); + } + } + + @Test + void expiredRemoteSessionRelocksUntilTheRenewedProofIsRedeemed() throws Exception { + Path codeFile = temporaryDirectory.resolve("renewed-unlock-code"); + MutableClock clock = new MutableClock(Instant.parse("2026-08-08T00:00:00Z")); + SetupRuntimeState state = new SetupRuntimeState(clock, mock(ManagedConfigCapability.class), + SetupPhase.CONFIGURATION_REQUIRED, SetupAccess.LOCKED, false, null); + try (SetupHttpUnlockService service = new SetupHttpUnlockService( + new RemoteSetupUnlock(codeFile, clock, new SecureRandom()), + InetAddress.getByName("0.0.0.0"), state, clock)) { + MockHttpServletRequest request = new MockHttpServletRequest("POST", "/api/setup/unlock"); + request.setRemoteAddr("198.51.100.4"); + service.redeem(new UnlockRequest(Files.readString(codeFile)), request); + assertThat(state.status().access()).isEqualTo(SetupAccess.UNLOCKED); + + clock.advance(Duration.ofMinutes(16)); + assertThat(service.requiresUnlock(request)).isTrue(); + assertThat(state.status().access()).isEqualTo(SetupAccess.LOCKED); + var renewed = service.redeem(new UnlockRequest(Files.readString(codeFile)), request); + + assertThat(renewed.response().access()).isEqualTo(SetupAccess.UNLOCKED); + assertThat(state.status().access()).isEqualTo(SetupAccess.UNLOCKED); + } + } + + @Test + void proofRenewalSerializesConcurrentRedemptionUntilRelockIsPublished() throws Exception { + Path codeFile = temporaryDirectory.resolve("serialized-renewal-code"); + MutableClock clock = new MutableClock(Instant.parse("2026-08-08T00:00:00Z")); + SetupRuntimeState state = new SetupRuntimeState(clock, mock(ManagedConfigCapability.class), + SetupPhase.CONFIGURATION_REQUIRED, SetupAccess.LOCKED, false, null); + try (SetupHttpUnlockService service = new SetupHttpUnlockService( + new RemoteSetupUnlock(codeFile, clock, new SecureRandom()), + InetAddress.getByName("0.0.0.0"), state, clock)) { + MockHttpServletRequest request = new MockHttpServletRequest("POST", "/api/setup/unlock"); + request.setRemoteAddr("198.51.100.7"); + service.redeem(new UnlockRequest(Files.readString(codeFile)), request); + clock.advance(Duration.ofMinutes(16)); + AtomicReference failure = new AtomicReference<>(); + AtomicReference exchange = new AtomicReference<>(); + Thread renew = new Thread(() -> { + try { + service.ensureProof(); + } catch (Throwable error) { + failure.compareAndSet(null, error); + } + }, "setup-proof-renew"); + Thread redeem; + + synchronized (state) { + renew.start(); + awaitBlocked(renew); + assertThat(Files.exists(codeFile)).isTrue(); + String renewedCode = Files.readString(codeFile); + redeem = new Thread(() -> { + try { + exchange.set(service.redeem(new UnlockRequest(renewedCode), request)); + } catch (Throwable error) { + failure.compareAndSet(null, error); + } + }, "setup-proof-redeem"); + redeem.start(); + awaitBlocked(redeem); + assertThat(Files.exists(codeFile)).isTrue(); + } + + renew.join(5_000); + redeem.join(5_000); + assertThat(renew.isAlive()).isFalse(); + assertThat(redeem.isAlive()).isFalse(); + assertThat(failure.get()).isNull(); + assertThat(state.status().access()).isEqualTo(SetupAccess.UNLOCKED); + assertThat(service.permits(exchange.get().cookie().getValue(), request)).isTrue(); + } + } + + private static void awaitBlocked(Thread thread) throws InterruptedException { + long deadline = System.nanoTime() + Duration.ofSeconds(5).toNanos(); + while (thread.getState() != Thread.State.BLOCKED && System.nanoTime() < deadline) { + Thread.sleep(10); + } + assertThat(thread.getState()).isEqualTo(Thread.State.BLOCKED); + } + + private static final class MutableClock extends Clock { + private Instant instant; + + private MutableClock(Instant instant) { + this.instant = instant; + } + + void advance(Duration duration) { + instant = instant.plus(duration); + } + + @Override + public ZoneId getZone() { + return ZoneOffset.UTC; + } + + @Override + public Clock withZone(ZoneId zone) { + return this; + } + + @Override + public Instant instant() { + return instant; + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/SetupPasswordFileLoaderTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/SetupPasswordFileLoaderTest.java new file mode 100644 index 0000000000..2a9c817c9b --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/SetupPasswordFileLoaderTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.unattended; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.mock.env.MockEnvironment; + +class SetupPasswordFileLoaderTest { + @TempDir + private Path temporaryDirectory; + + @Test + void readsOnlyOwnerFileAndClearsScopedCopy() throws Exception { + Path file = temporaryDirectory.resolve("administrator-password"); + Files.write(file, "file-only-secret\n".getBytes(StandardCharsets.UTF_8)); + Files.setPosixFilePermissions(file, PosixFilePermissions.fromString("rw-------")); + SetupPasswordFileLoader loader = new SetupPasswordFileLoader(); + + try (SetupPasswordFileLoader.Password password = loader.read(file)) { + char[] copy = password.copy(); + assertArrayEquals("file-only-secret".toCharArray(), copy); + java.util.Arrays.fill(copy, '\0'); + assertTrue(password.toString().contains("redacted")); + } + } + + @Test + void rejectsPlainPropertyEvenWhenPasswordFileIsAlsoPresent() { + MockEnvironment environment = new MockEnvironment().withProperty( + "hertzbeat.setup.administrator.password", "forbidden") + .withProperty("hertzbeat.setup.administrator.password-file", "/run/secrets/admin"); + + assertThrows(IllegalStateException.class, () -> SetupPasswordFileLoader.requireFilePath( + environment, "hertzbeat.setup.administrator")); + } + + @Test + void rejectsNonOwnerFileAndSymlink() throws Exception { + Path file = temporaryDirectory.resolve("password"); + Files.writeString(file, "secret"); + Files.setPosixFilePermissions(file, PosixFilePermissions.fromString("rw-r--r--")); + SetupPasswordFileLoader loader = new SetupPasswordFileLoader(); + assertThrows(IllegalStateException.class, () -> loader.read(file)); + + Path target = temporaryDirectory.resolve("target"); + Files.writeString(target, "secret"); + Files.setPosixFilePermissions(target, PosixFilePermissions.fromString("rw-------")); + Path link = temporaryDirectory.resolve("link"); + Files.createSymbolicLink(link, target); + assertThrows(IllegalStateException.class, () -> loader.read(link)); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializerTest.java new file mode 100644 index 0000000000..14ec1674ea --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializerTest.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.unattended; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; +import java.time.Instant; +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigSource; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ManagementDatabaseSummary; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionalConfigurationSummary; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupAccess; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.StatusResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreSummary; +import org.apache.hertzbeat.manager.setup.workflow.HeadlessSetupWorkflow; +import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransitionScheduler; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.mock.env.MockEnvironment; + +class UnattendedSetupInitializerTest { + @TempDir + private Path temporaryDirectory; + + @Test + void completedRestartIsIdempotentAndPerformsNoWrites() { + HeadlessSetupWorkflow workflow = mock(HeadlessSetupWorkflow.class); + when(workflow.status()).thenReturn(status(SetupPhase.COMPLETE)); + MockEnvironment environment = new MockEnvironment().withProperty( + UnattendedSetupInitializer.ENABLED_PROPERTY, "true"); + + new UnattendedSetupInitializer(workflow, environment, new SetupPasswordFileLoader()).initialize(); + + verify(workflow, never()).configure(any()); + verify(workflow, never()).createAdministrator(any(), any()); + verify(workflow, never()).complete(any()); + } + + @Test + void disabledInitializationDoesNotEvenReadSetupStatus() { + HeadlessSetupWorkflow workflow = mock(HeadlessSetupWorkflow.class); + new UnattendedSetupInitializer(workflow, new MockEnvironment(), + new SetupPasswordFileLoader()).initialize(); + verify(workflow, never()).status(); + } + + @Test + void administratorPhaseConsumesPasswordFileAndCompletesSetup() throws Exception { + Path passwordFile = temporaryDirectory.resolve("administrator-password"); + Files.writeString(passwordFile, "owner-secret\n"); + Files.setPosixFilePermissions(passwordFile, PosixFilePermissions.fromString("rw-------")); + HeadlessSetupWorkflow workflow = mock(HeadlessSetupWorkflow.class); + SetupRuntimeTransitionScheduler transitions = mock(SetupRuntimeTransitionScheduler.class); + when(workflow.status()).thenReturn(status(SetupPhase.ADMINISTRATOR_REQUIRED)); + MockEnvironment environment = new MockEnvironment() + .withProperty(UnattendedSetupInitializer.ENABLED_PROPERTY, "true") + .withProperty("hertzbeat.setup.unattended.acknowledged-warnings", "h2_non_production") + .withProperty("hertzbeat.setup.administrator.username", "operator") + .withProperty("hertzbeat.setup.administrator.password-file", passwordFile.toString()); + + new UnattendedSetupInitializer(workflow, environment, new SetupPasswordFileLoader(), Optional.of(transitions)) + .initialize(); + + verify(workflow).createAdministrator(eq("operator"), any()); + verify(workflow).complete(java.util.List.of(SetupWarningCode.H2_NON_PRODUCTION)); + verify(transitions).installationCompleted(); + } + + @Test + void explicitlyConfirmsTheSameH2AndPlainHttpWarningsAsBrowserSetup() { + HeadlessSetupWorkflow workflow = mock(HeadlessSetupWorkflow.class); + when(workflow.status()).thenReturn(status(SetupPhase.OPTIONAL_CONFIGURATION)); + MockEnvironment environment = new MockEnvironment() + .withProperty(UnattendedSetupInitializer.ENABLED_PROPERTY, "true") + .withProperty("hertzbeat.setup.unattended.acknowledged-warnings", + "h2_non_production, public_address_plaintext"); + + new UnattendedSetupInitializer(workflow, environment, new SetupPasswordFileLoader()).initialize(); + + verify(workflow).complete(java.util.List.of( + SetupWarningCode.H2_NON_PRODUCTION, SetupWarningCode.PUBLIC_ADDRESS_PLAINTEXT)); + } + + private static StatusResponse status(SetupPhase phase) { + return new StatusResponse(phase, Instant.parse("2026-08-08T00:00:00Z"), SetupAccess.LOCAL, + ApplyMode.MANAGED_WRITE, true, null, null, + new ManagementDatabaseSummary(MetadataDatabaseKind.H2, true, ConfigSource.UI_MANAGED, false), + new TelemetryStoreSummary(TelemetryStoreKind.GREPTIME, true, ConfigSource.UI_MANAGED, false), + phase != SetupPhase.ADMINISTRATOR_REQUIRED, + new OptionalConfigurationSummary(false, false, false, false, false)); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java new file mode 100644 index 0000000000..de86398a95 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.time.Clock; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.AdministratorRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.CompleteRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.PublicAccessConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupAccess; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidateRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiException; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigurationBundle; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.apache.hertzbeat.manager.setup.identity.IdentityInitializationService; +import org.junit.jupiter.api.Test; + +class DefaultSetupWorkflowTest { + + @Test + void wrongPhaseMustBeRejectedBeforeAdministratorOrCompletionWrites() { + ManagedConfigCapability capability = mock(ManagedConfigCapability.class); + SetupRuntimeState state = new SetupRuntimeState(Clock.systemUTC(), capability, + SetupPhase.CONFIGURATION_REQUIRED, SetupAccess.LOCAL, false, null); + IdentityInitializationService identities = mock(IdentityInitializationService.class); + SetupCompletionCoordinator completion = mock(SetupCompletionCoordinator.class); + SetupMutationSerializer mutations = new SetupMutationSerializer(); + DefaultSetupWorkflow workflow = new DefaultSetupWorkflow(state, mock(SetupRequestValidator.class), + mock(SetupConfigurationCoordinator.class), mock(SetupOperationRegistry.class), capability, + Optional.of(identities), Optional.of(completion), + mock(SetupOptionsCoordinator.class), Clock.systemUTC(), mutations); + + assertThrows(SetupWorkflowConflict.class, + () -> workflow.createAdministrator(new AdministratorRequest("operator", "secret"))); + assertThrows(SetupWorkflowConflict.class, + () -> workflow.complete(new CompleteRequest(SetupPhase.OPTIONAL_CONFIGURATION, List.of()))); + + verifyNoInteractions(identities, completion); + } + + @Test + void completionRequiresEveryPendingWarningAcknowledgement() { + ManagedConfigCapability capability = mock(ManagedConfigCapability.class); + SetupRuntimeState state = new SetupRuntimeState(Clock.systemUTC(), capability, + SetupPhase.OPTIONAL_CONFIGURATION, SetupAccess.LOCAL, true, "operator"); + SetupCompletionCoordinator completion = mock(SetupCompletionCoordinator.class); + SetupMutationSerializer mutations = new SetupMutationSerializer(); + DefaultSetupWorkflow workflow = new DefaultSetupWorkflow(state, mock(SetupRequestValidator.class), + mock(SetupConfigurationCoordinator.class), mock(SetupOperationRegistry.class), capability, + Optional.of(mock(IdentityInitializationService.class)), Optional.of(completion), + mock(SetupOptionsCoordinator.class), Clock.systemUTC(), mutations); + + assertThrows(SetupApiException.class, + () -> workflow.complete(new CompleteRequest(SetupPhase.OPTIONAL_CONFIGURATION, List.of()))); + verifyNoInteractions(completion); + + workflow.complete(new CompleteRequest(SetupPhase.OPTIONAL_CONFIGURATION, + List.of(SetupWarningCode.H2_NON_PRODUCTION))); + org.mockito.Mockito.verify(completion).completeInstallation(); + } + + @Test + void completionCannotCommitAgainstWarningsThatOptionsPersistenceIsStillPublishing() throws Exception { + ManagedConfigCapability capability = mock(ManagedConfigCapability.class); + SetupRuntimeState state = new SetupRuntimeState(Clock.systemUTC(), capability, + SetupPhase.OPTIONAL_CONFIGURATION, SetupAccess.LOCAL, true, "operator"); + SetupRequestValidator validator = mock(SetupRequestValidator.class); + when(validator.validate(any(ValidateRequest.class))) + .thenReturn(new ValidationResponse(true, Instant.now(), null, List.of())); + SetupOptionsCoordinator options = mock(SetupOptionsCoordinator.class); + CountDownLatch persistenceStarted = new CountDownLatch(1); + CountDownLatch allowPersistence = new CountDownLatch(1); + doAnswer(invocation -> { + persistenceStarted.countDown(); + allowPersistence.await(5, TimeUnit.SECONDS); + return null; + }).when(options).persist(any()); + SetupCompletionCoordinator completion = mock(SetupCompletionCoordinator.class); + SetupMutationSerializer mutations = new SetupMutationSerializer(); + DefaultSetupWorkflow workflow = new DefaultSetupWorkflow(state, validator, + mock(SetupConfigurationCoordinator.class), mock(SetupOperationRegistry.class), capability, + Optional.of(mock(IdentityInitializationService.class)), Optional.of(completion), + options, Clock.systemUTC(), mutations); + HeadlessSetupCoordinator headless = new HeadlessSetupCoordinator(state, validator, + mock(SetupConfigurationCoordinator.class), capability, + Optional.of(mock(IdentityInitializationService.class)), Optional.of(completion), mutations); + OptionsRequest request = new OptionsRequest( + new PublicAccessConfiguration("http://localhost:1157", null, null), null, null); + + try (var executor = Executors.newFixedThreadPool(2)) { + var optionsResult = executor.submit(() -> workflow.configureOptions(request)); + persistenceStarted.await(5, TimeUnit.SECONDS); + var completionResult = executor.submit(() -> headless.complete( + List.of(SetupWarningCode.H2_NON_PRODUCTION))); + + assertThrows(TimeoutException.class, () -> completionResult.get(200, TimeUnit.MILLISECONDS)); + allowPersistence.countDown(); + optionsResult.get(5, TimeUnit.SECONDS); + ExecutionException rejected = assertThrows(ExecutionException.class, + () -> completionResult.get(5, TimeUnit.SECONDS)); + org.assertj.core.api.Assertions.assertThat(rejected.getCause()).isInstanceOf(SetupApiException.class); + } + verifyNoInteractions(completion); + } + + @Test + void runningBrowserConfigurationPreventsHeadlessConfigurationFromCrossingThePhaseTransition() + throws Exception { + ManagedConfigCapability capability = mock(ManagedConfigCapability.class); + SetupRuntimeState state = new SetupRuntimeState(Clock.systemUTC(), capability, + SetupPhase.CONFIGURATION_REQUIRED, SetupAccess.LOCAL, false, null); + SetupRequestValidator validator = mock(SetupRequestValidator.class); + when(validator.validate(any(ValidateRequest.class))) + .thenReturn(new ValidationResponse(true, Instant.now(), null, List.of())); + SetupConfigurationCoordinator configuration = mock(SetupConfigurationCoordinator.class); + CountDownLatch configurationStarted = new CountDownLatch(1); + CountDownLatch allowConfiguration = new CountDownLatch(1); + doAnswer(invocation -> { + configurationStarted.countDown(); + allowConfiguration.await(5, TimeUnit.SECONDS); + return new ConfigurationResponse("operation-1", SetupOperationState.AWAITING_RESTART, + SetupPhase.APPLICATION_STARTING, 0, false); + }).when(configuration).configure(any(ConfigurationRequest.class), eq(capability)); + SetupMutationSerializer mutations = new SetupMutationSerializer(); + DefaultSetupWorkflow browser = new DefaultSetupWorkflow(state, validator, configuration, + mock(SetupOperationRegistry.class), capability, + Optional.of(mock(IdentityInitializationService.class)), + Optional.of(mock(SetupCompletionCoordinator.class)), mock(SetupOptionsCoordinator.class), + Clock.systemUTC(), mutations); + HeadlessSetupCoordinator headless = new HeadlessSetupCoordinator(state, validator, configuration, + capability, Optional.of(mock(IdentityInitializationService.class)), + Optional.of(mock(SetupCompletionCoordinator.class)), mutations); + ConfigurationRequest browserRequest = new ConfigurationRequest( + SetupPhase.CONFIGURATION_REQUIRED, ApplyMode.MANAGED_WRITE, + new MetadataDatabaseConfiguration(MetadataDatabaseKind.H2, + "jdbc:h2:mem:browser", "sa", "secret"), + new TelemetryStoreConfiguration(TelemetryStoreKind.GREPTIME, + "localhost:4001", "http://localhost:4000", "public", null, null)); + + try (SecretValue metadataPassword = SecretValue.of("secret"); + var executor = Executors.newFixedThreadPool(2)) { + var browserResult = executor.submit(() -> browser.configure(browserRequest)); + configurationStarted.await(5, TimeUnit.SECONDS); + var headlessResult = executor.submit(() -> headless.configure( + new HeadlessSetupWorkflow.RequiredConfiguration(ApplyMode.MANAGED_WRITE, + new HeadlessSetupWorkflow.Metadata(MetadataDatabaseKind.H2, + "jdbc:h2:mem:headless", "sa", metadataPassword), + new HeadlessSetupWorkflow.Telemetry("localhost:4001", + "http://localhost:4000", "public", Optional.empty(), Optional.empty())))); + + assertThrows(TimeoutException.class, () -> headlessResult.get(200, TimeUnit.MILLISECONDS)); + allowConfiguration.countDown(); + browserResult.get(5, TimeUnit.SECONDS); + ExecutionException rejected = assertThrows(ExecutionException.class, + () -> headlessResult.get(5, TimeUnit.SECONDS)); + org.assertj.core.api.Assertions.assertThat(rejected.getCause()) + .isInstanceOf(SetupWorkflowConflict.class); + } + verify(configuration, never()).configure(any(HeadlessSetupWorkflow.RequiredConfiguration.class), + any(ManagedConfigurationBundle.class), eq(capability)); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/GreptimeHttpConnectionProbeTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/GreptimeHttpConnectionProbeTest.java new file mode 100644 index 0000000000..d943af0731 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/GreptimeHttpConnectionProbeTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.sun.net.httpserver.HttpServer; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreKind; +import org.junit.jupiter.api.Test; + +class GreptimeHttpConnectionProbeTest { + + @Test + void probeExecutesAuthenticatedBoundedSqlQueryAgainstSelectedDatabase() throws Exception { + HttpServer server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + AtomicReference observed = new AtomicReference<>(); + server.createContext("/v1/sql", exchange -> { + observed.set(exchange.getRequestMethod() + " " + exchange.getRequestURI() + " " + + exchange.getRequestHeaders().getFirst("Authorization") + " " + + new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.US_ASCII)); + exchange.sendResponseHeaders(200, -1); + exchange.close(); + }); + server.start(); + try { + String endpoint = "http://" + InetAddress.getLoopbackAddress().getHostAddress() + + ":" + server.getAddress().getPort(); + TelemetryStoreConfiguration configuration = new TelemetryStoreConfiguration( + TelemetryStoreKind.GREPTIME, "localhost:4001", endpoint, + "public", "telemetry", "secret"); + + assertThat(new GreptimeHttpConnectionProbe(Duration.ofSeconds(3)).probe(configuration)).isEmpty(); + assertThat(observed.get()).isEqualTo( + "POST /v1/sql?db=public Basic dGVsZW1ldHJ5OnNlY3JldA== sql=SELECT%201"); + } finally { + server.stop(0); + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinatorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinatorTest.java new file mode 100644 index 0000000000..92fd844800 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinatorTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +import java.time.Clock; +import java.util.List; +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionalConfigurationSummary; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupAccess; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiException; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; +import org.apache.hertzbeat.manager.setup.identity.IdentityInitializationService; +import org.junit.jupiter.api.Test; + +class HeadlessSetupCoordinatorTest { + @Test + void completionRequiresTheSamePendingWarningAcknowledgementsAsBrowserSetup() { + ManagedConfigCapability capability = mock(ManagedConfigCapability.class); + SetupRuntimeState state = new SetupRuntimeState(Clock.systemUTC(), capability, + SetupPhase.OPTIONAL_CONFIGURATION, SetupAccess.LOCAL, true, "operator"); + state.optionsConfigured(new OptionalConfigurationSummary(true, false, false, false, false), + List.of(SetupWarningCode.H2_NON_PRODUCTION, + SetupWarningCode.PUBLIC_ADDRESS_PLAINTEXT)); + SetupCompletionCoordinator completion = mock(SetupCompletionCoordinator.class); + HeadlessSetupCoordinator coordinator = new HeadlessSetupCoordinator(state, + mock(SetupRequestValidator.class), mock(SetupConfigurationCoordinator.class), capability, + Optional.of(mock(IdentityInitializationService.class)), Optional.of(completion), + new SetupMutationSerializer()); + + assertThrows(SetupApiException.class, + () -> coordinator.complete(List.of(SetupWarningCode.H2_NON_PRODUCTION))); + verifyNoInteractions(completion); + + coordinator.complete(List.of(SetupWarningCode.H2_NON_PRODUCTION, + SetupWarningCode.PUBLIC_ADDRESS_PLAINTEXT)); + verify(completion).completeInstallation(); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JakartaMailConnectionProbeTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JakartaMailConnectionProbeTest.java new file mode 100644 index 0000000000..7b5192b299 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JakartaMailConnectionProbeTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.concurrent.Executors; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; +import org.junit.jupiter.api.Test; + +class JakartaMailConnectionProbeTest { + + @Test + void probeOpensAndClosesRealSmtpTransportWithoutSendingMail() throws Exception { + try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getLoopbackAddress()); + var executor = Executors.newSingleThreadExecutor()) { + var conversation = executor.submit(() -> serveSmtp(server)); + MailConfiguration configuration = new MailConfiguration( + InetAddress.getLoopbackAddress().getHostAddress(), server.getLocalPort(), + MailSecurity.NONE, null, null, "hertzbeat@example.test"); + + assertThat(new JakartaMailConnectionProbe(Duration.ofSeconds(3)).probe(configuration)).isEmpty(); + assertThat(conversation.get()).startsWith("EHLO "); + } + } + + private static String serveSmtp(ServerSocket server) throws Exception { + try (var socket = server.accept(); + var reader = new BufferedReader(new InputStreamReader( + socket.getInputStream(), StandardCharsets.US_ASCII)); + var writer = new BufferedWriter(new OutputStreamWriter( + socket.getOutputStream(), StandardCharsets.US_ASCII))) { + writer.write("220 localhost setup probe\r\n"); + writer.flush(); + String greeting = reader.readLine(); + writer.write("250 localhost\r\n"); + writer.flush(); + String quit = reader.readLine(); + if ("QUIT".equals(quit)) { + writer.write("221 bye\r\n"); + writer.flush(); + } + return greeting; + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataConnectionProbeTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataConnectionProbeTest.java new file mode 100644 index 0000000000..07123e7077 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataConnectionProbeTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.contains; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.junit.jupiter.api.Test; + +class JdbcMetadataConnectionProbeTest { + + @Test + void h2ProbePerformsRealConnectionAndTemporaryDdlDmlCleanup() throws Exception { + String url = "jdbc:h2:mem:setup_validation;DB_CLOSE_DELAY=-1"; + var configuration = new MetadataDatabaseConfiguration( + MetadataDatabaseKind.H2, url, "sa", "password"); + + assertThat(new JdbcMetadataConnectionProbe(Duration.ofSeconds(30)).probe(configuration)).isEmpty(); + try (var connection = java.sql.DriverManager.getConnection(url, "sa", "password"); + var result = connection.getMetaData().getTables(null, null, "HZB_SETUP_PROBE_%", null)) { + assertThat(result.next()).isFalse(); + } + } + + @Test + void productMismatchHasStableSchemaError() { + var configuration = new MetadataDatabaseConfiguration( + MetadataDatabaseKind.MYSQL, "jdbc:h2:mem:setup_wrong_kind", "sa", "password"); + + assertThat(new JdbcMetadataConnectionProbe(Duration.ofSeconds(10)).probe(configuration)) + .contains(SetupErrorCode.METADATA_SCHEMA_MISMATCH); + } + + @Test + void blockingDriverCannotGrowProbeThreadsOrQueueWithoutBound() { + ThreadPoolExecutor executor = new ThreadPoolExecutor(2, 2, 0L, TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(1), Thread.ofPlatform().name("bounded-probe-", 0).factory(), + new ThreadPoolExecutor.AbortPolicy()); + CountDownLatch release = new CountDownLatch(1); + JdbcMetadataConnectionProbe probe = new JdbcMetadataConnectionProbe( + Duration.ofMillis(100), executor, (url, username, password) -> { + while (release.getCount() > 0) { + try { + release.await(); + } catch (InterruptedException ignored) { + // Emulate a driver that ignores interruption while connecting. + } + } + throw new java.sql.SQLException("released"); + }); + var configuration = new MetadataDatabaseConfiguration( + MetadataDatabaseKind.H2, "jdbc:h2:mem:blocked", "sa", "password"); + List>> calls = new ArrayList<>(); + try { + for (int index = 0; index < 8; index++) { + calls.add(CompletableFuture.supplyAsync(() -> probe.probe(configuration))); + } + calls.forEach(call -> assertThat(call.join()).contains(SetupErrorCode.METADATA_CONNECTION_FAILED)); + assertThat(executor.getLargestPoolSize()).isEqualTo(2); + assertThat(executor.getQueue().size()).isLessThanOrEqualTo(1); + } finally { + release.countDown(); + executor.shutdownNow(); + } + } + + @Test + void successfulTransactionalDdlDropsBeforeCommitWithoutRollback() throws Exception { + Connection connection = mock(Connection.class); + Statement statement = mock(Statement.class); + ResultSet result = mock(ResultSet.class); + when(connection.createStatement()).thenReturn(statement); + when(statement.executeQuery(contains("SELECT probe_value"))).thenReturn(result); + when(result.next()).thenReturn(true); + when(result.getString(1)).thenReturn("updated"); + + assertThat(new JdbcMetadataConnectionProbe(Duration.ofSeconds(1)) + .validatePrivileges(connection)).isEmpty(); + + var order = inOrder(statement, connection); + order.verify(statement).execute(contains("CREATE TABLE")); + order.verify(statement).execute(contains("DROP TABLE")); + order.verify(connection).commit(); + org.mockito.Mockito.verify(connection, org.mockito.Mockito.never()).rollback(); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationCoordinatorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationCoordinatorTest.java new file mode 100644 index 0000000000..a145334e59 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationCoordinatorTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Path; +import java.time.Clock; +import java.util.Arrays; +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreKind; +import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigDeploymentDetector; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.DeploymentConstraint; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class SetupConfigurationCoordinatorTest { + @TempDir + private Path installationRoot; + + @Test + void managedWritePublishesOneRecoverablePairAndRecordsRestartOperation() { + SetupOperationRegistry operations = new SetupOperationRegistry(Clock.systemUTC()); + SetupConfigurationCoordinator coordinator = new SetupConfigurationCoordinator( + new ManagedConfigurationTransaction(installationRoot), operations); + + var response = coordinator.configure(request(ApplyMode.MANAGED_WRITE), + new ManagedConfigDeploymentDetector(installationRoot).detect()); + + assertEquals(SetupOperationState.AWAITING_RESTART, response.state()); + assertEquals(SetupPhase.APPLICATION_STARTING, response.phase()); + assertFalse(response.exportAvailable()); + assertEquals(response.operationId(), operations.get(response.operationId()).operationId()); + assertEquals(ManagedActiveConfigurationInspector.State.LOADABLE, + new ManagedActiveConfigurationInspector(installationRoot).inspect().state()); + } + + @Test + void externalApplyDoesNotWriteAndMakesExportAvailable() { + SetupOperationRegistry operations = new SetupOperationRegistry(Clock.systemUTC()); + SetupConfigurationCoordinator coordinator = new SetupConfigurationCoordinator( + new ManagedConfigurationTransaction(installationRoot), operations); + ManagedConfigCapability capability = new ManagedConfigDeploymentDetector(installationRoot).detect(); + + var response = coordinator.configure(request(ApplyMode.EXTERNAL_APPLY), + new ManagedConfigCapability(ApplyMode.EXTERNAL_APPLY, false, + DeploymentConstraint.READ_ONLY)); + + assertEquals(SetupOperationState.AWAITING_EXTERNAL_APPLY, response.state()); + assertEquals(SetupPhase.EXTERNAL_APPLY_REQUIRED, response.phase()); + assertTrue(response.exportAvailable()); + assertEquals(ManagedActiveConfigurationInspector.State.ABSENT, + new ManagedActiveConfigurationInspector(installationRoot).inspect().state()); + assertThrows(SetupWorkflowConflict.class, + () -> coordinator.configure(request(ApplyMode.EXTERNAL_APPLY), capability)); + } + + @Test + void headlessExternalApplyClosesTheCoordinatorOwnedSecretBundle() { + SetupConfigurationCoordinator coordinator = new SetupConfigurationCoordinator( + new ManagedConfigurationTransaction(installationRoot), + new SetupOperationRegistry(Clock.systemUTC())); + try (SecretValue callerPassword = SecretValue.of("metadata-password")) { + var request = new HeadlessSetupWorkflow.RequiredConfiguration(ApplyMode.EXTERNAL_APPLY, + new HeadlessSetupWorkflow.Metadata(MetadataDatabaseKind.H2, + "jdbc:h2:./data/setup", "sa", callerPassword), + new HeadlessSetupWorkflow.Telemetry("localhost:4001", "http://localhost:4000", + "public", Optional.empty(), Optional.empty())); + var bundle = SetupConfigurationMapper.map(request); + + coordinator.configure(request, bundle, + new ManagedConfigCapability(ApplyMode.EXTERNAL_APPLY, false, + DeploymentConstraint.READ_ONLY)); + + assertArrayEquals(new char["metadata-password".length()], + bundle.secrets().metadataDatabasePassword().copy()); + char[] retained = callerPassword.copy(); + try { + assertArrayEquals("metadata-password".toCharArray(), retained); + } finally { + Arrays.fill(retained, '\0'); + } + } + } + + private static ConfigurationRequest request(ApplyMode applyMode) { + return new ConfigurationRequest(SetupPhase.CONFIGURATION_REQUIRED, applyMode, + new MetadataDatabaseConfiguration(MetadataDatabaseKind.H2, + "jdbc:h2:./data/setup", "sa", "metadata-password"), + new TelemetryStoreConfiguration(TelemetryStoreKind.GREPTIME, + "localhost:4001", "http://localhost:4000", "public", null, null)); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationMapperTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationMapperTest.java new file mode 100644 index 0000000000..92bbf0d153 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationMapperTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.junit.jupiter.api.Test; + +class SetupConfigurationMapperTest { + @Test + void headlessMappingCopiesCallerOwnedSecretsIntoCoordinatorOwnedBundle() { + SecretValue metadata = SecretValue.of("metadata-secret"); + SecretValue telemetry = SecretValue.of("telemetry-secret"); + var request = new HeadlessSetupWorkflow.RequiredConfiguration(ApplyMode.MANAGED_WRITE, + new HeadlessSetupWorkflow.Metadata(MetadataDatabaseKind.H2, + "jdbc:h2:./data/setup", "sa", metadata), + new HeadlessSetupWorkflow.Telemetry("localhost:4001", "http://localhost:4000", + "public", Optional.of("greptime"), Optional.of(telemetry))); + + var mapped = SetupConfigurationMapper.map(request); + metadata.close(); + telemetry.close(); + + assertThat(mapped.secrets().metadataDatabasePassword().copy()) + .containsExactly("metadata-secret".toCharArray()); + assertThat(mapped.secrets().telemetryPassword().orElseThrow().copy()) + .containsExactly("telemetry-secret".toCharArray()); + mapped.close(); + assertThat(mapped.secrets().metadataDatabasePassword().copy()).containsOnly('\0'); + assertThat(mapped.secrets().telemetryPassword().orElseThrow().copy()).containsOnly('\0'); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidatorProbeTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidatorProbeTest.java new file mode 100644 index 0000000000..fe32e87f10 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidatorProbeTest.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.time.Clock; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidateRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationSection; +import org.apache.hertzbeat.manager.setup.api.SetupApiException; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.junit.jupiter.api.Test; + +class SetupRequestValidatorProbeTest { + + @Test + void sectionProbesAreExplicitAndTheirStableFailureIsReturned() { + MetadataConnectionProbe metadata = ignored -> Optional.of(SetupErrorCode.METADATA_SCHEMA_MISMATCH); + TelemetryConnectionProbe telemetry = ignored -> Optional.empty(); + MailConnectionProbe mail = ignored -> Optional.empty(); + SetupRequestValidator validator = new SetupRequestValidator( + Clock.systemUTC(), metadata, telemetry, mail); + + var response = validator.validate(new ValidateRequest(ValidationSection.METADATA_DATABASE, + metadata(), null, null, null)); + + assertThat(response.valid()).isFalse(); + assertThat(response.errorCode()).isEqualTo(SetupErrorCode.METADATA_SCHEMA_MISMATCH); + } + + @Test + void structuralFailuresDoNotReachConnectionProbe() { + MetadataConnectionProbe metadata = ignored -> { + throw new AssertionError("connection probe must not be called"); + }; + SetupRequestValidator validator = new SetupRequestValidator( + Clock.systemUTC(), metadata, ignored -> Optional.empty(), ignored -> Optional.empty()); + MetadataDatabaseConfiguration wrongKind = new MetadataDatabaseConfiguration( + MetadataDatabaseKind.MYSQL, "jdbc:h2:mem:wrong", "sa", "password"); + + var response = validator.validate(new ValidateRequest(ValidationSection.METADATA_DATABASE, + wrongKind, null, null, null)); + + assertThat(response.errorCode()).isEqualTo(SetupErrorCode.METADATA_KIND_UNSUPPORTED); + } + + @Test + void telemetryAndMailUseTheirConnectionBoundaries() { + SetupRequestValidator validator = new SetupRequestValidator(Clock.systemUTC(), + ignored -> Optional.empty(), + ignored -> Optional.of(SetupErrorCode.TELEMETRY_CONNECTION_FAILED), + ignored -> Optional.of(SetupErrorCode.MAIL_CONNECTION_FAILED)); + + assertThat(validator.validate(new ValidateRequest(ValidationSection.TELEMETRY_STORE, + null, telemetry(), null, null)).errorCode()) + .isEqualTo(SetupErrorCode.TELEMETRY_CONNECTION_FAILED); + assertThat(validator.validate(new ValidateRequest(ValidationSection.MAIL, + null, null, null, mail())).errorCode()) + .isEqualTo(SetupErrorCode.MAIL_CONNECTION_FAILED); + } + + @Test + void headlessTelemetryValidatesCredentialPairWithoutHttpSecretDto() { + TelemetryConnectionProbe telemetry = ignored -> { + throw new AssertionError("connection probe must not be called"); + }; + SetupRequestValidator validator = new SetupRequestValidator(Clock.systemUTC(), + ignored -> Optional.empty(), telemetry, ignored -> Optional.empty()); + try (SecretValue password = SecretValue.of("secret")) { + var configuration = new HeadlessSetupWorkflow.Telemetry( + "localhost:4001", "http://localhost:4000", "public", + Optional.empty(), Optional.of(password)); + + assertThrows(SetupApiException.class, () -> validator.validate(configuration)); + } + } + + @Test + void headlessMetadataProbeUsesAndClearsAnIndependentSecretOwner() { + AtomicReference observed = new AtomicReference<>(); + MetadataConnectionProbe metadata = request -> { + observed.set(request); + assertThat(request.password().copy()).containsExactly("secret".toCharArray()); + return Optional.empty(); + }; + SetupRequestValidator validator = new SetupRequestValidator(Clock.systemUTC(), metadata, + ignored -> Optional.empty(), ignored -> Optional.empty()); + try (SecretValue callerSecret = SecretValue.of("secret")) { + validator.validate(new HeadlessSetupWorkflow.Metadata(MetadataDatabaseKind.H2, + "jdbc:h2:mem:setup-probe", "sa", callerSecret)); + + assertThat(observed.get().password().copy()).containsOnly('\0'); + assertThat(callerSecret.copy()).containsExactly("secret".toCharArray()); + } + } + + @Test + void headlessTelemetryProbeUsesAndClearsAnIndependentSecretOwner() { + AtomicReference observed = new AtomicReference<>(); + TelemetryConnectionProbe telemetry = request -> { + observed.set(request); + assertThat(request.password().orElseThrow().copy()).containsExactly("secret".toCharArray()); + return Optional.empty(); + }; + SetupRequestValidator validator = new SetupRequestValidator(Clock.systemUTC(), + ignored -> Optional.empty(), telemetry, ignored -> Optional.empty()); + try (SecretValue callerSecret = SecretValue.of("secret")) { + validator.validate(new HeadlessSetupWorkflow.Telemetry( + "localhost:4001", "http://localhost:4000", "public", + Optional.of("telemetry"), Optional.of(callerSecret))); + + assertThat(observed.get().password().orElseThrow().copy()).containsOnly('\0'); + assertThat(callerSecret.copy()).containsExactly("secret".toCharArray()); + } + } + + @Test + void headlessMetadataUsesTheSameKindAndJdbcStructureValidatorAsBrowserRequests() { + MetadataConnectionProbe metadata = ignored -> { + throw new AssertionError("connection probe must not be called"); + }; + SetupRequestValidator validator = new SetupRequestValidator(Clock.systemUTC(), metadata, + ignored -> Optional.empty(), ignored -> Optional.empty()); + try (SecretValue password = SecretValue.of("secret")) { + SetupApiException failure = assertThrows(SetupApiException.class, + () -> validator.validate(new HeadlessSetupWorkflow.Metadata(MetadataDatabaseKind.MYSQL, + "jdbc:h2:mem:wrong-kind", "sa", password))); + + assertThat(failure.errorCode()).isEqualTo(SetupErrorCode.METADATA_KIND_UNSUPPORTED); + } + } + + private static MetadataDatabaseConfiguration metadata() { + return new MetadataDatabaseConfiguration( + MetadataDatabaseKind.H2, "jdbc:h2:mem:setup-probe", "sa", "password"); + } + + private static TelemetryStoreConfiguration telemetry() { + return new TelemetryStoreConfiguration(TelemetryStoreKind.GREPTIME, + "localhost:4001", "http://localhost:4000", "public", null, null); + } + + private static MailConfiguration mail() { + return new MailConfiguration("localhost", 2525, MailSecurity.NONE, + null, null, "hertzbeat@example.test"); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidatorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidatorTest.java new file mode 100644 index 0000000000..c50ac3d549 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidatorTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.PublicAccessConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidateRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationSection; +import org.junit.jupiter.api.Test; + +class SetupRequestValidatorTest { + private final Clock clock = Clock.fixed(Instant.parse("2026-08-08T00:00:00Z"), ZoneOffset.UTC); + private final SetupRequestValidator validator = new SetupRequestValidator(clock); + + @Test + void metadataKindMustMatchJdbcSchemeWithoutReturningConnectionDetails() { + var response = validator.validate(new ValidateRequest(ValidationSection.METADATA_DATABASE, + new MetadataDatabaseConfiguration(MetadataDatabaseKind.POSTGRESQL, + "jdbc:mysql://db/hertzbeat", "user", "password"), null, null, null)); + + assertFalse(response.valid()); + assertEquals(SetupErrorCode.METADATA_KIND_UNSUPPORTED, response.errorCode()); + assertEquals(clock.instant(), response.observedAt()); + assertFalse(response.toString().contains("jdbc:mysql")); + } + + @Test + void publicAddressValidatorProducesStablePlaintextWarning() { + var response = validator.validate(new ValidateRequest(ValidationSection.PUBLIC_ACCESS, + null, null, new PublicAccessConfiguration("http://monitor.example.test", null, null), null)); + + assertTrue(response.valid()); + assertEquals(1, response.warnings().size()); + } + + @Test + void publicGrpcPortMustFitTheTransportRange() { + var response = validator.validate(new ValidateRequest(ValidationSection.PUBLIC_ACCESS, + null, null, new PublicAccessConfiguration(null, null, "collector.example.test:99999"), null)); + + assertFalse(response.valid()); + assertEquals(SetupErrorCode.PUBLIC_ADDRESS_INVALID, response.errorCode()); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupWarningPolicyTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupWarningPolicyTest.java new file mode 100644 index 0000000000..80d20cec86 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupWarningPolicyTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.PublicAccessConfiguration; +import org.junit.jupiter.api.Test; + +class SetupWarningPolicyTest { + @Test + void liveAndRestartInputsProduceTheSameWarnings() { + var options = new OptionsRequest( + new PublicAccessConfiguration("http://localhost:1157", null, null), null, + new MailConfiguration("localhost", 25, MailSecurity.NONE, null, null, "ops@example.test")); + assertThat(SetupWarningPolicy.INSTANCE.evaluate(MetadataDatabaseKind.H2, options)) + .containsExactlyElementsOf(SetupWarningPolicy.INSTANCE.evaluate( + MetadataDatabaseKind.H2, "http://localhost:1157", MailSecurity.NONE)); + } +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/bootstrap/SetupOnlyApplication.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/bootstrap/SetupOnlyApplication.java index c2ce5800fa..fe357049ba 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/bootstrap/SetupOnlyApplication.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/bootstrap/SetupOnlyApplication.java @@ -18,6 +18,7 @@ package org.apache.hertzbeat.bootstrap; import org.apache.hertzbeat.common.runtime.BusinessRuntimeConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiConfiguration; import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeAccessConfiguration; import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.actuate.autoconfigure.endpoint.EndpointAutoConfiguration; @@ -45,6 +46,6 @@ import org.springframework.context.annotation.Import; HealthEndpointAutoConfiguration.class, WebMvcHealthEndpointExtensionAutoConfiguration.class }) -@Import({BusinessRuntimeConfiguration.class, SetupRuntimeAccessConfiguration.class}) +@Import({BusinessRuntimeConfiguration.class, SetupRuntimeAccessConfiguration.class, SetupApiConfiguration.class}) public class SetupOnlyApplication { } diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/HertzBeatApplication.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/HertzBeatApplication.java index abc095c775..50bf2e5827 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/HertzBeatApplication.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/HertzBeatApplication.java @@ -21,6 +21,7 @@ import jakarta.annotation.PostConstruct; import org.apache.hertzbeat.bootstrap.SetupOnlyApplication; import org.apache.hertzbeat.manager.nativex.HertzbeatRuntimeHintsRegistrar; import org.apache.hertzbeat.startup.runtime.HertzBeatStartupCoordinator; +import org.apache.hertzbeat.startup.runtime.LocalInstallationStartupProbe; import org.apache.hertzbeat.startup.runtime.SpringStartupContextLauncher; import org.apache.hertzbeat.startup.runtime.StartupModePropertyProbe; import org.springframework.boot.SpringBootConfiguration; @@ -55,7 +56,7 @@ public class HertzBeatApplication { public static void main(String[] args) { SpringStartupContextLauncher launcher = new SpringStartupContextLauncher(); HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( - new StartupModePropertyProbe(), launcher); + new StartupModePropertyProbe(new LocalInstallationStartupProbe()), launcher); coordinator.start(args); } diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/config/ManagedConfigEnvironmentPostProcessor.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/config/ManagedConfigEnvironmentPostProcessor.java index 3bc6efdb42..57e6e5c008 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/config/ManagedConfigEnvironmentPostProcessor.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/config/ManagedConfigEnvironmentPostProcessor.java @@ -23,6 +23,7 @@ import org.apache.hertzbeat.common.runtime.RuntimeMode; import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector; import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector.Inspection; import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector.State; +import org.apache.hertzbeat.manager.setup.config.SetupInstallationPaths; import org.springframework.boot.EnvironmentPostProcessor; import org.springframework.boot.SpringApplication; import org.springframework.boot.context.config.ConfigDataEnvironmentPostProcessor; @@ -42,7 +43,7 @@ import org.springframework.core.io.Resource; /** Loads the two fixed managed files between operator files and classpath defaults for every profile. */ public final class ManagedConfigEnvironmentPostProcessor implements EnvironmentPostProcessor, Ordered { - static final String INSTALLATION_ROOT_PROPERTY = "hertzbeat.internal.installation-root"; + static final String INSTALLATION_ROOT_PROPERTY = SetupInstallationPaths.ROOT_PROPERTY; public static final String INTERNAL_RUNTIME_PROPERTY_SOURCE = "hertzbeatInternalRuntimeMode"; private static final String DEFAULT_INSTALLATION_ROOT = "."; diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java index 440a23e07f..d6a3569201 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java @@ -46,6 +46,12 @@ public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition return transition(decision); } + @Override + public synchronized void configurationApplied() { + transition(new StartupDecision(RuntimeMode.FULL_SETUP_GATED, + SetupPhase.ADMINISTRATOR_REQUIRED, null)); + } + @Override public synchronized void completeSetup() { transition(new StartupDecision(RuntimeMode.NORMAL, SetupPhase.COMPLETE, null)); diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbe.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbe.java new file mode 100644 index 0000000000..54f170efe4 --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbe.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup.runtime; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.security.SecureRandom; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector; +import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector.State; +import org.apache.hertzbeat.manager.setup.config.SetupInstallationPaths; +import org.apache.hertzbeat.manager.setup.installation.LocalInstallationFingerprintStore; + +/** Filesystem-first startup convergence with an explicit legacy/external upgrade entry. */ +public final class LocalInstallationStartupProbe implements StartupDecisionProbe { + private final Path root; + private final boolean externalDatabaseConfigured; + + public LocalInstallationStartupProbe() { + this(Path.of(System.getProperty(SetupInstallationPaths.ROOT_PROPERTY, ".")), + externalDatabaseConfigured()); + } + + LocalInstallationStartupProbe(Path root, boolean externalDatabaseConfigured) { + this.root = root.toAbsolutePath().normalize(); + this.externalDatabaseConfigured = externalDatabaseConfigured; + } + + @Override + public StartupDecision probe() { + State managed = new ManagedActiveConfigurationInspector(root).inspect().state(); + if (managed == State.RECOVERY_REQUIRED) { + return StartupDecision.recovery(); + } + FingerprintState fingerprint = fingerprintState(); + if (fingerprint == FingerprintState.INVALID) { + return StartupDecision.recovery(); + } + boolean legacyDatabase = legacyH2Present(); + if (fingerprint == FingerprintState.PRESENT) { + return managed == State.LOADABLE || legacyDatabase || externalDatabaseConfigured + ? new StartupDecision(RuntimeMode.FULL_SETUP_GATED, + SetupPhase.ADMINISTRATOR_REQUIRED, null) : StartupDecision.recovery(); + } + if (managed == State.LOADABLE || legacyDatabase || externalDatabaseConfigured) { + return new StartupDecision(RuntimeMode.FULL_SETUP_GATED, + SetupPhase.ADMINISTRATOR_REQUIRED, null); + } + return new StartupDecision(RuntimeMode.SETUP_ONLY, SetupPhase.CONFIGURATION_REQUIRED, null); + } + + private FingerprintState fingerprintState() { + Path path = root.resolve("data/config/.installation-fingerprint"); + try { + boolean present = new LocalInstallationFingerprintStore(path, new SecureRandom()).read().isPresent(); + if (present) { + return FingerprintState.PRESENT; + } + return Files.exists(path, LinkOption.NOFOLLOW_LINKS) + ? FingerprintState.INVALID : FingerprintState.ABSENT; + } catch (IOException failure) { + return FingerprintState.INVALID; + } + } + + private boolean legacyH2Present() { + return Files.isRegularFile(root.resolve("data/hertzbeat.mv.db")) + || Files.isRegularFile(root.resolve("data/hertzbeat.h2.db")); + } + + private static boolean externalDatabaseConfigured() { + return hasText(System.getProperty("spring.datasource.url")) + || hasText(System.getenv("SPRING_DATASOURCE_URL")); + } + + private static boolean hasText(String value) { + return value != null && !value.isBlank(); + } + + private enum FingerprintState { ABSENT, PRESENT, INVALID } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbeTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbeTest.java new file mode 100644 index 0000000000..cd3061fce0 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbeTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup.runtime; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.SecureRandom; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.setup.config.GreptimeEndpoints; +import org.apache.hertzbeat.manager.setup.config.GreptimeSettings; +import org.apache.hertzbeat.manager.setup.config.ManagedApplicationConfig; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigurationBundle; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.ManagedSecrets; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.apache.hertzbeat.manager.setup.installation.LocalInstallationFingerprintStore; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class LocalInstallationStartupProbeTest { + @TempDir + private Path root; + + @Test + void freshRootStartsSetupOnlyAndLegacyDatabaseStartsGated() throws Exception { + assertEquals(RuntimeMode.SETUP_ONLY, new LocalInstallationStartupProbe(root, false).probe().mode()); + Files.createDirectories(root.resolve("data")); + Files.createFile(root.resolve("data/hertzbeat.mv.db")); + assertEquals(RuntimeMode.FULL_SETUP_GATED, + new LocalInstallationStartupProbe(root, false).probe().mode()); + } + + @Test + void localFingerprintCanNeverOpenBusinessRuntimeWithoutDatabaseComparison() throws Exception { + new ManagedConfigurationTransaction(root).apply(bundle()); + assertEquals(RuntimeMode.FULL_SETUP_GATED, + new LocalInstallationStartupProbe(root, false).probe().mode()); + new LocalInstallationFingerprintStore(root.resolve("data/config/.installation-fingerprint"), + new SecureRandom()).create(); + assertEquals(RuntimeMode.FULL_SETUP_GATED, + new LocalInstallationStartupProbe(root, false).probe().mode()); + } + + private static ManagedConfigurationBundle bundle() { + ManagedApplicationConfig application = new ManagedApplicationConfig( + new MetadataDatabaseSettings( + org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind.H2, + "jdbc:h2:./data/hertzbeat", "sa"), + GreptimeSettings.anonymous(new GreptimeEndpoints( + "localhost:4001", "http://localhost:4000"), "public")); + return new ManagedConfigurationBundle(application, + ManagedSecrets.withoutTelemetryPassword(SecretValue.of("password"))); + } +} From 733bc5e93f2530b44de146355b059c46bb2d8cde Mon Sep 17 00:00:00 2001 From: Logic Date: Sat, 8 Aug 2026 21:57:11 +0800 Subject: [PATCH 12/71] Unify setup lifecycle transitions --- .../setup/api/SetupApiConfiguration.java | 37 ++-- .../manager/setup/api/SetupApiContract.java | 2 + .../setup/api/SetupExceptionHandler.java | 9 +- .../LoggingRecoveryFailureReporter.java | 22 ++ .../config/ManagedConfigurationRecovery.java | 16 +- .../ManagedConfigurationTransaction.java | 35 ++- .../setup/config/RecoveryFailureReporter.java | 37 ++++ .../UnattendedSetupInitializer.java | 3 +- .../setup/workflow/DefaultSetupWorkflow.java | 67 +----- .../workflow/HeadlessSetupCoordinator.java | 68 +----- .../setup/workflow/HeadlessSetupWorkflow.java | 5 +- .../SetupConfigurationCoordinator.java | 20 +- .../workflow/SetupOperationRegistry.java | 9 + .../setup/workflow/SetupRuntimeState.java | 6 +- .../workflow/SetupTransitionService.java | 197 +++++++++++++++++ .../setup/api/SetupApiContractTest.java | 3 +- .../setup/api/SetupControllerTest.java | 16 +- .../ManagedConfigurationTransactionTest.java | 34 ++- .../workflow/DefaultSetupWorkflowTest.java | 37 +++- .../HeadlessSetupCoordinatorTest.java | 7 +- .../SetupConfigurationCoordinatorTest.java | 31 ++- .../SetupConfigurationMapperTest.java | 4 +- .../workflow/SetupTransitionServiceTest.java | 209 ++++++++++++++++++ 23 files changed, 699 insertions(+), 175 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/LoggingRecoveryFailureReporter.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/RecoveryFailureReporter.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionService.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionServiceTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java index 30d7ff631f..4a880fd415 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java @@ -55,6 +55,7 @@ import org.apache.hertzbeat.manager.setup.workflow.SetupOptionsCoordinator; import org.apache.hertzbeat.manager.setup.workflow.HeadlessSetupWorkflow; import org.apache.hertzbeat.manager.setup.workflow.SetupRequestValidator; import org.apache.hertzbeat.manager.setup.workflow.SetupRuntimeState; +import org.apache.hertzbeat.manager.setup.workflow.SetupTransitionService; import org.springframework.beans.factory.ObjectProvider; import org.springframework.boot.ApplicationRunner; import org.springframework.context.annotation.Bean; @@ -132,31 +133,31 @@ public class SetupApiConfiguration { return new SetupMutationSerializer(); } + @Bean + public SetupTransitionService setupTransitionService( + SetupRuntimeState state, SetupRequestValidator validator, + SetupConfigurationCoordinator configuration, ManagedConfigCapability capability, + ObjectProvider identityProvider, + ObjectProvider installationProvider, Environment environment) { + return new SetupTransitionService(state, validator, configuration, capability, + identityProvider.stream().findFirst(), + completion(environment, installationProvider.stream().findFirst())); + } + @Bean public DefaultSetupWorkflow setupWorkflow(Environment environment, SetupRuntimeState state, - SetupRequestValidator validator, SetupConfigurationCoordinator configuration, - SetupOperationRegistry operations, ManagedConfigCapability capability, - ObjectProvider identityProvider, - ObjectProvider installationProvider, - SetupMutationSerializer mutations) { - Optional completion = completion( - environment, installationProvider.stream().findFirst()); - return new DefaultSetupWorkflow(state, validator, configuration, operations, capability, - identityProvider.stream().findFirst(), completion, + SetupRequestValidator validator, + SetupOperationRegistry operations, + SetupMutationSerializer mutations, SetupTransitionService transitions) { + return new DefaultSetupWorkflow(state, validator, operations, new SetupOptionsCoordinator(new ManagedConfigurationTransaction( - SetupInstallationPaths.root(environment))), Clock.systemUTC(), mutations); + SetupInstallationPaths.root(environment))), Clock.systemUTC(), mutations, transitions); } @Bean public HeadlessSetupWorkflow headlessSetupWorkflow( - Environment environment, SetupRuntimeState state, SetupRequestValidator validator, - SetupConfigurationCoordinator configuration, ManagedConfigCapability capability, - ObjectProvider identityProvider, - ObjectProvider installationProvider, - SetupMutationSerializer mutations) { - return new HeadlessSetupCoordinator(state, validator, configuration, capability, - identityProvider.stream().findFirst(), - completion(environment, installationProvider.stream().findFirst()), mutations); + SetupRuntimeState state, SetupMutationSerializer mutations, SetupTransitionService transitions) { + return new HeadlessSetupCoordinator(state, mutations, transitions); } @Bean diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java index 824c4a75a9..c1784e9386 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java @@ -234,6 +234,8 @@ public final class SetupApiContract { SETUP_CODE_EXPIRED("setup_code_expired"), SETUP_RATE_LIMITED("setup_rate_limited"), SETUP_NOT_COMPLETE("setup_not_complete"), + INVALID_REQUEST("invalid_request"), + INTERNAL_ERROR("internal_error"), CONFIG_READ_ONLY("config_read_only"), CONFIG_WRITE_FAILED("config_write_failed"), CONFIG_RECOVERY_REQUIRED("config_recovery_required"), diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandler.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandler.java index 279852a4cd..d14798b3b9 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandler.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandler.java @@ -17,7 +17,6 @@ package org.apache.hertzbeat.manager.setup.api; -import jakarta.servlet.http.HttpServletRequest; import java.time.Clock; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; import org.apache.hertzbeat.manager.setup.security.SetupUnlockRejected; @@ -62,15 +61,13 @@ public class SetupExceptionHandler { } @ExceptionHandler({MethodArgumentNotValidException.class, HttpMessageNotReadableException.class}) - public ResponseEntity invalidRequest(Exception ignored, HttpServletRequest request) { - SetupErrorCode code = SetupApiContract.UNLOCK_PATH.equals(request.getRequestURI()) - ? SetupErrorCode.SETUP_CODE_INVALID : SetupErrorCode.OPERATION_CONFLICT; - return response(HttpStatus.BAD_REQUEST, code); + public ResponseEntity invalidRequest(Exception ignored) { + return response(HttpStatus.BAD_REQUEST, SetupErrorCode.INVALID_REQUEST); } @ExceptionHandler(Exception.class) public ResponseEntity unexpectedFailure(Exception ignored) { - return response(HttpStatus.INTERNAL_SERVER_ERROR, SetupErrorCode.CONFIG_WRITE_FAILED); + return response(HttpStatus.INTERNAL_SERVER_ERROR, SetupErrorCode.INTERNAL_ERROR); } private ResponseEntity response(HttpStatus status, SetupErrorCode code) { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/LoggingRecoveryFailureReporter.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/LoggingRecoveryFailureReporter.java new file mode 100644 index 0000000000..f039c2f6ff --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/LoggingRecoveryFailureReporter.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Production adapter that logs only the pre-sanitized recovery diagnostic event. */ +final class LoggingRecoveryFailureReporter implements RecoveryFailureReporter { + private static final Logger LOGGER = LoggerFactory.getLogger(LoggingRecoveryFailureReporter.class); + + @Override + public void report(Failure failure) { + LOGGER.warn("Managed configuration recovery failure stage={} store={} exception={} detail={}", + failure.stage(), failure.store(), failure.exceptionClass(), failure.safeMessage()); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationRecovery.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationRecovery.java index 4895f4e45d..f0d540feca 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationRecovery.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationRecovery.java @@ -13,11 +13,13 @@ import java.io.IOException; final class ManagedConfigurationRecovery { private final ManagedApplicationConfigStore applicationStore; private final ManagedSecretStore secretStore; + private final RecoveryFailureReporter reporter; ManagedConfigurationRecovery(ManagedApplicationConfigStore applicationStore, - ManagedSecretStore secretStore) { + ManagedSecretStore secretStore, RecoveryFailureReporter reporter) { this.applicationStore = applicationStore; this.secretStore = secretStore; + this.reporter = reporter; } ManagedConfigurationTransaction.Outcome recover() { @@ -58,11 +60,15 @@ final class ManagedConfigurationRecovery { try { applicationStore.discardCandidate(); } catch (IOException failure) { + reporter.report(RecoveryFailureReporter.Stage.DISCARD_CANDIDATE, + RecoveryFailureReporter.Store.APPLICATION, failure); discarded = false; } try { secretStore.discardCandidate(); } catch (IOException failure) { + reporter.report(RecoveryFailureReporter.Stage.DISCARD_CANDIDATE, + RecoveryFailureReporter.Store.SECRET, failure); discarded = false; } return discarded; @@ -103,6 +109,8 @@ final class ManagedConfigurationRecovery { applicationStore.promoteCandidate(candidate.value().orElseThrow(), candidate.generation().orElseThrow()); return true; } catch (IOException failure) { + reporter.report(RecoveryFailureReporter.Stage.PROMOTE_CANDIDATE, + RecoveryFailureReporter.Store.APPLICATION, failure); return false; } } @@ -112,6 +120,8 @@ final class ManagedConfigurationRecovery { secretStore.promoteCandidate(candidate.value().orElseThrow(), candidate.generation().orElseThrow()); return true; } catch (IOException failure) { + reporter.report(RecoveryFailureReporter.Stage.PROMOTE_CANDIDATE, + RecoveryFailureReporter.Store.SECRET, failure); return false; } } @@ -121,6 +131,8 @@ final class ManagedConfigurationRecovery { applicationStore.restoreActive(candidate); return true; } catch (IOException failure) { + reporter.report(RecoveryFailureReporter.Stage.RESTORE_ACTIVE, + RecoveryFailureReporter.Store.APPLICATION, failure); return false; } } @@ -130,6 +142,8 @@ final class ManagedConfigurationRecovery { secretStore.restoreActive(candidate); return true; } catch (IOException failure) { + reporter.report(RecoveryFailureReporter.Stage.RESTORE_ACTIVE, + RecoveryFailureReporter.Store.SECRET, failure); return false; } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransaction.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransaction.java index 6eecd48ec4..1274e808f0 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransaction.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransaction.java @@ -37,20 +37,31 @@ public final class ManagedConfigurationTransaction { private final ManagedSecretStore secretStore; private final Path lockFile; private final ManagedConfigurationRecovery recovery; + private final RecoveryFailureReporter reporter; /** Creates the production file transaction rooted at the HertzBeat installation. */ public ManagedConfigurationTransaction(Path installationRoot) { this(new FileManagedApplicationConfigStore(installationRoot), - new FileManagedSecretStore(installationRoot), installationRoot); + new FileManagedSecretStore(installationRoot), installationRoot, + new LoggingRecoveryFailureReporter()); } ManagedConfigurationTransaction( ManagedApplicationConfigStore applicationStore, ManagedSecretStore secretStore, Path installationRoot) { + this(applicationStore, secretStore, installationRoot, new LoggingRecoveryFailureReporter()); + } + + ManagedConfigurationTransaction( + ManagedApplicationConfigStore applicationStore, + ManagedSecretStore secretStore, + Path installationRoot, + RecoveryFailureReporter reporter) { this.applicationStore = Objects.requireNonNull(applicationStore, "applicationStore"); this.secretStore = Objects.requireNonNull(secretStore, "secretStore"); - this.recovery = new ManagedConfigurationRecovery(applicationStore, secretStore); + this.reporter = Objects.requireNonNull(reporter, "reporter"); + this.recovery = new ManagedConfigurationRecovery(applicationStore, secretStore, reporter); Path root = Objects.requireNonNull(installationRoot, "installationRoot") .toAbsolutePath().normalize(); this.lockFile = root.resolve("data/config").resolve(LOCK_FILE); @@ -103,14 +114,22 @@ public final class ManagedConfigurationTransaction { } try { applicationStore.promoteCandidate(bundle.application(), generation); - secretStore.promoteCandidate(bundle.secrets(), generation); - if (!matchesExpectedActive(bundle, generation)) { - return recovery.rollback(previousApplication, previousSecrets); - } - return Outcome.APPLIED; - } catch (IOException ignored) { + } catch (IOException failure) { + reporter.report(RecoveryFailureReporter.Stage.PROMOTE_CANDIDATE, + RecoveryFailureReporter.Store.APPLICATION, failure); return recovery.rollback(previousApplication, previousSecrets); } + try { + secretStore.promoteCandidate(bundle.secrets(), generation); + } catch (IOException failure) { + reporter.report(RecoveryFailureReporter.Stage.PROMOTE_CANDIDATE, + RecoveryFailureReporter.Store.SECRET, failure); + return recovery.rollback(previousApplication, previousSecrets); + } + if (!matchesExpectedActive(bundle, generation)) { + return recovery.rollback(previousApplication, previousSecrets); + } + return Outcome.APPLIED; } finally { close(previousSecrets); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/RecoveryFailureReporter.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/RecoveryFailureReporter.java new file mode 100644 index 0000000000..f7c0941df1 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/RecoveryFailureReporter.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +/** Secret-free diagnostic boundary for managed configuration recovery failures. */ +@FunctionalInterface +public interface RecoveryFailureReporter { + String SAFE_MESSAGE = "Managed configuration recovery operation failed"; + + void report(Failure failure); + + default void report(Stage stage, Store store, Exception failure) { + report(new Failure(stage, store, failure.getClass().getName(), SAFE_MESSAGE)); + } + + /** Recovery operation stage that failed. */ + enum Stage { + DISCARD_CANDIDATE, + PROMOTE_CANDIDATE, + RESTORE_ACTIVE + } + + /** Managed aggregate member involved in the failure. */ + enum Store { + APPLICATION, + SECRET + } + + /** Fully sanitized diagnostic event safe for production logging. */ + record Failure(Stage stage, Store store, String exceptionClass, String safeMessage) { + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializer.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializer.java index 2dfaa62b92..d92c614467 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializer.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializer.java @@ -112,7 +112,8 @@ public final class UnattendedSetupInitializer { private void configure(StatusResponse status, HeadlessSetupWorkflow.Metadata metadata, HeadlessSetupWorkflow.Telemetry telemetry) { var response = workflow.configure( - new HeadlessSetupWorkflow.RequiredConfiguration(status.applyMode(), metadata, telemetry)); + new HeadlessSetupWorkflow.RequiredConfiguration( + status.phase(), status.applyMode(), metadata, telemetry)); if (response.phase() == SetupPhase.APPLICATION_STARTING) { transitions.ifPresent(SetupRuntimeTransitionScheduler::configurationApplied); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflow.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflow.java index 6d1d527774..19b8d7a616 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflow.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflow.java @@ -18,8 +18,6 @@ package org.apache.hertzbeat.manager.setup.workflow; import java.time.Clock; -import java.util.Arrays; -import java.util.Optional; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.AdministratorRequest; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.AdministratorResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.CompleteRequest; @@ -42,40 +40,29 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationRespons import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationSection; import org.apache.hertzbeat.manager.setup.api.SetupApiException; import org.apache.hertzbeat.manager.setup.api.SetupWorkflow; -import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; -import org.apache.hertzbeat.manager.setup.identity.AdministratorCredentials; -import org.apache.hertzbeat.manager.setup.identity.BootstrapIdentityConflict; -import org.apache.hertzbeat.manager.setup.identity.IdentityInitializationService; import org.springframework.http.HttpStatus; /** Cohesive setup state-machine facade; transport and persistence remain in dedicated collaborators. */ public final class DefaultSetupWorkflow implements SetupWorkflow { private final SetupRuntimeState state; private final SetupRequestValidator validator; - private final SetupConfigurationCoordinator configuration; private final SetupOperationRegistry operations; - private final ManagedConfigCapability capability; - private final Optional identities; - private final Optional completion; private final SetupOptionsCoordinator options; private final Clock clock; private final SetupMutationSerializer mutations; + private final SetupTransitionService transitions; public DefaultSetupWorkflow(SetupRuntimeState state, SetupRequestValidator validator, - SetupConfigurationCoordinator configuration, SetupOperationRegistry operations, - ManagedConfigCapability capability, Optional identities, - Optional completion, SetupOptionsCoordinator options, - Clock clock, SetupMutationSerializer mutations) { + SetupOperationRegistry operations, + SetupOptionsCoordinator options, Clock clock, SetupMutationSerializer mutations, + SetupTransitionService transitions) { this.state = state; this.validator = validator; - this.configuration = configuration; this.operations = operations; - this.capability = capability; - this.identities = identities; - this.completion = completion; this.options = options; this.clock = clock; this.mutations = mutations; + this.transitions = transitions; } @Override @@ -101,15 +88,7 @@ public final class DefaultSetupWorkflow implements SetupWorkflow { } private ConfigurationResponse configureMutation(ConfigurationRequest request) { - requireWritable(); - state.ensurePhase(SetupPhase.CONFIGURATION_REQUIRED); - requireValid(new ValidateRequest(ValidationSection.METADATA_DATABASE, - request.managementDatabase(), null, null, null)); - requireValid(new ValidateRequest(ValidationSection.TELEMETRY_STORE, - null, request.telemetryStore(), null, null)); - ConfigurationResponse response = configuration.configure(request, capability); - state.configurationApplied(response.operationId(), response.phase()); - return response; + return transitions.configure(SetupTransitionService.ConfigurationCommand.browser(request)); } @Override @@ -123,19 +102,9 @@ public final class DefaultSetupWorkflow implements SetupWorkflow { } private AdministratorResponse createAdministratorMutation(AdministratorRequest request) { - requireWritable(); - state.ensurePhase(SetupPhase.ADMINISTRATOR_REQUIRED); - char[] password = request.password().toCharArray(); - try { - identities.orElseThrow(SetupWorkflowConflict::new) - .createFirstAdministrator(new AdministratorCredentials(request.username(), password)); - } catch (BootstrapIdentityConflict conflict) { - throw new SetupApiException(SetupErrorCode.ADMINISTRATOR_ALREADY_CONFIGURED, HttpStatus.CONFLICT); - } finally { - Arrays.fill(password, '\0'); - } - state.administratorCreated(request.username()); - return new AdministratorResponse(request.username(), SetupPhase.OPTIONAL_CONFIGURATION); + String username = transitions.createAdministrator( + SetupTransitionService.AdministratorCommand.browser(request)); + return new AdministratorResponse(username, SetupPhase.OPTIONAL_CONFIGURATION); } @Override @@ -183,22 +152,8 @@ public final class DefaultSetupWorkflow implements SetupWorkflow { } private CompleteResponse completeMutation(CompleteRequest request) { - requireWritable(); - state.ensurePhase(SetupPhase.OPTIONAL_CONFIGURATION); - if (request.expectedPhase() != SetupPhase.OPTIONAL_CONFIGURATION) { - throw new SetupWorkflowConflict(); - } - if (!request.acknowledgedWarnings().containsAll(state.pendingWarnings())) { - throw new SetupApiException(SetupErrorCode.OPERATION_CONFLICT, HttpStatus.CONFLICT); - } - String username = state.administratorUsername(); - if (username == null) { - throw new SetupWorkflowConflict(); - } - completion.orElseThrow(SetupWorkflowConflict::new).completeInstallation(); - state.complete(); - CompleteResponse response = new CompleteResponse(SetupPhase.COMPLETE, clock.instant(), "/login", username); - return response; + String username = transitions.complete(SetupTransitionService.CompletionCommand.browser(request)); + return new CompleteResponse(SetupPhase.COMPLETE, clock.instant(), "/login", username); } private void requireWritable() { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinator.java index 629d02134d..db452f89cd 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinator.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinator.java @@ -7,45 +7,23 @@ package org.apache.hertzbeat.manager.setup.workflow; -import java.util.Arrays; import java.util.List; -import java.util.Optional; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationResponse; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.StatusResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; -import org.apache.hertzbeat.manager.setup.api.SetupApiException; -import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; import org.apache.hertzbeat.manager.setup.config.SecretValue; -import org.apache.hertzbeat.manager.setup.identity.AdministratorCredentials; -import org.apache.hertzbeat.manager.setup.identity.BootstrapIdentityConflict; -import org.apache.hertzbeat.manager.setup.identity.IdentityInitializationService; -import org.springframework.http.HttpStatus; /** Executes non-HTTP setup commands while retaining clearable secret ownership. */ public final class HeadlessSetupCoordinator implements HeadlessSetupWorkflow { private final SetupRuntimeState state; - private final SetupRequestValidator validator; - private final SetupConfigurationCoordinator configuration; - private final ManagedConfigCapability capability; - private final Optional identities; - private final Optional completion; private final SetupMutationSerializer mutations; + private final SetupTransitionService transitions; - public HeadlessSetupCoordinator(SetupRuntimeState state, SetupRequestValidator validator, - SetupConfigurationCoordinator configuration, - ManagedConfigCapability capability, - Optional identities, - Optional completion, - SetupMutationSerializer mutations) { + public HeadlessSetupCoordinator(SetupRuntimeState state, SetupMutationSerializer mutations, + SetupTransitionService transitions) { this.state = state; - this.validator = validator; - this.configuration = configuration; - this.capability = capability; - this.identities = identities; - this.completion = completion; this.mutations = mutations; + this.transitions = transitions; } @Override @@ -59,14 +37,7 @@ public final class HeadlessSetupCoordinator implements HeadlessSetupWorkflow { } private ConfigurationResponse configureMutation(RequiredConfiguration request) { - requireWritable(); - state.ensurePhase(SetupPhase.CONFIGURATION_REQUIRED); - validator.validate(request.metadata()); - validator.validate(request.telemetry()); - ConfigurationResponse response = configuration.configure( - request, SetupConfigurationMapper.map(request), capability); - state.configurationApplied(response.operationId(), response.phase()); - return response; + return transitions.configure(SetupTransitionService.ConfigurationCommand.headless(request)); } @Override @@ -75,17 +46,7 @@ public final class HeadlessSetupCoordinator implements HeadlessSetupWorkflow { } private void createAdministratorMutation(String username, SecretValue password) { - requireWritable(); - state.ensurePhase(SetupPhase.ADMINISTRATOR_REQUIRED); - char[] clear = password.copy(); - try (AdministratorCredentials credentials = new AdministratorCredentials(username, clear)) { - identities.orElseThrow(SetupWorkflowConflict::new).createFirstAdministrator(credentials); - } catch (BootstrapIdentityConflict conflict) { - throw new SetupApiException(SetupErrorCode.ADMINISTRATOR_ALREADY_CONFIGURED, HttpStatus.CONFLICT); - } finally { - Arrays.fill(clear, '\0'); - } - state.administratorCreated(username); + transitions.createAdministrator(SetupTransitionService.AdministratorCommand.headless(username, password)); } @Override @@ -94,21 +55,6 @@ public final class HeadlessSetupCoordinator implements HeadlessSetupWorkflow { } private void completeMutation(List acknowledgedWarnings) { - requireWritable(); - state.ensurePhase(SetupPhase.OPTIONAL_CONFIGURATION); - if (!acknowledgedWarnings.containsAll(state.pendingWarnings())) { - throw new SetupApiException(SetupErrorCode.OPERATION_CONFLICT, HttpStatus.CONFLICT); - } - if (state.administratorUsername() == null) { - throw new SetupWorkflowConflict(); - } - completion.orElseThrow(SetupWorkflowConflict::new).completeInstallation(); - state.complete(); - } - - private void requireWritable() { - if (state.phase() == SetupPhase.COMPLETE) { - throw new SetupApiException(SetupErrorCode.SETUP_COMPLETE, HttpStatus.GONE); - } + transitions.complete(SetupTransitionService.CompletionCommand.headless(acknowledgedWarnings)); } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupWorkflow.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupWorkflow.java index ae1f7ba98f..4e8e4fa9a4 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupWorkflow.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupWorkflow.java @@ -23,6 +23,7 @@ import java.util.Optional; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.StatusResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; import org.apache.hertzbeat.manager.setup.config.SecretValue; @@ -38,8 +39,10 @@ public interface HeadlessSetupWorkflow { void complete(List acknowledgedWarnings); /** Required managed configuration with secret values kept in clearable owners. */ - record RequiredConfiguration(ApplyMode applyMode, Metadata metadata, Telemetry telemetry) { + record RequiredConfiguration(SetupPhase expectedPhase, ApplyMode applyMode, + Metadata metadata, Telemetry telemetry) { public RequiredConfiguration { + Objects.requireNonNull(expectedPhase, "expectedPhase"); Objects.requireNonNull(applyMode, "applyMode"); Objects.requireNonNull(metadata, "metadata"); Objects.requireNonNull(telemetry, "telemetry"); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationCoordinator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationCoordinator.java index a25578f64c..1ca187b151 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationCoordinator.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationCoordinator.java @@ -42,11 +42,10 @@ public final class SetupConfigurationCoordinator { } public ConfigurationResponse configure(ConfigurationRequest request, ManagedConfigCapability capability) { - if (request.expectedPhase() != SetupPhase.CONFIGURATION_REQUIRED - || request.applyMode() != capability.applyMode()) { + if (!configurationPhase(request.expectedPhase()) || request.applyMode() != capability.applyMode()) { throw new SetupWorkflowConflict(); } - String operationId = operations.begin(SetupPhase.CONFIGURATION_REQUIRED); + String operationId = beginOperation(request.expectedPhase()); if (request.applyMode() == ApplyMode.EXTERNAL_APPLY) { operations.finish(operationId, SetupOperationState.AWAITING_EXTERNAL_APPLY, SetupPhase.EXTERNAL_APPLY_REQUIRED, null, true); @@ -65,10 +64,10 @@ public final class SetupConfigurationCoordinator { ManagedConfigurationBundle bundle, ManagedConfigCapability capability) { try (bundle) { - if (request.applyMode() != capability.applyMode()) { + if (!configurationPhase(request.expectedPhase()) || request.applyMode() != capability.applyMode()) { throw new SetupWorkflowConflict(); } - String operationId = operations.begin(SetupPhase.CONFIGURATION_REQUIRED); + String operationId = beginOperation(request.expectedPhase()); if (request.applyMode() == ApplyMode.EXTERNAL_APPLY) { operations.finish(operationId, SetupOperationState.AWAITING_EXTERNAL_APPLY, SetupPhase.EXTERNAL_APPLY_REQUIRED, null, true); @@ -113,4 +112,15 @@ public final class SetupConfigurationCoordinator { return new ConfigurationResponse(operation.operationId(), operation.state(), operation.phase(), operation.nextPollAfterMillis(), operation.exportAvailable()); } + + private static boolean configurationPhase(SetupPhase phase) { + // External apply deliberately supports explicit re-entry after refresh; no submitted secret is retained. + return phase == SetupPhase.CONFIGURATION_REQUIRED || phase == SetupPhase.EXTERNAL_APPLY_REQUIRED; + } + + private String beginOperation(SetupPhase expectedPhase) { + return expectedPhase == SetupPhase.EXTERNAL_APPLY_REQUIRED + ? operations.replaceExternalApply(SetupPhase.CONFIGURATION_REQUIRED) + : operations.begin(SetupPhase.CONFIGURATION_REQUIRED); + } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOperationRegistry.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOperationRegistry.java index 5ab58fa3b6..a222cb4f5a 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOperationRegistry.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOperationRegistry.java @@ -52,6 +52,15 @@ public final class SetupOperationRegistry { return id; } + public synchronized String replaceExternalApply(SetupPhase phase) { + if (activeOperationId == null + || operations.get(activeOperationId).state() != SetupOperationState.AWAITING_EXTERNAL_APPLY) { + throw new SetupWorkflowConflict(); + } + activeOperationId = null; + return begin(phase); + } + public synchronized OperationResponse finish( String id, SetupOperationState state, SetupPhase phase, SetupErrorCode errorCode, boolean exportAvailable) { OperationResponse current = require(id); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupRuntimeState.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupRuntimeState.java index e351fd47ea..66b182b374 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupRuntimeState.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupRuntimeState.java @@ -95,7 +95,11 @@ public final class SetupRuntimeState { } public synchronized void configurationApplied(String id, SetupPhase next) { - requirePhase(SetupPhase.CONFIGURATION_REQUIRED); + configurationApplied(SetupPhase.CONFIGURATION_REQUIRED, id, next); + } + + public synchronized void configurationApplied(SetupPhase expected, String id, SetupPhase next) { + requirePhase(expected); operationId = id; phase = next; } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionService.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionService.java new file mode 100644 index 0000000000..154e6a8e3c --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionService.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.AdministratorRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.CompleteRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidateRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationSection; +import org.apache.hertzbeat.manager.setup.api.SetupApiException; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.apache.hertzbeat.manager.setup.identity.AdministratorCredentials; +import org.apache.hertzbeat.manager.setup.identity.BootstrapIdentityConflict; +import org.apache.hertzbeat.manager.setup.identity.IdentityInitializationService; +import org.springframework.http.HttpStatus; + +/** Single state transition boundary shared by browser and headless setup adapters. */ +public final class SetupTransitionService { + private final SetupRuntimeState state; + private final SetupRequestValidator validator; + private final SetupConfigurationCoordinator configuration; + private final ManagedConfigCapability capability; + private final Optional identities; + private final Optional completion; + + public SetupTransitionService(SetupRuntimeState state, SetupRequestValidator validator, + SetupConfigurationCoordinator configuration, ManagedConfigCapability capability, + Optional identities, + Optional completion) { + this.state = state; + this.validator = validator; + this.configuration = configuration; + this.capability = capability; + this.identities = identities; + this.completion = completion; + } + + public ConfigurationResponse configure(ConfigurationCommand command) { + requireWritable(); + SetupPhase expected = command.expectedPhase(); + if (expected != SetupPhase.CONFIGURATION_REQUIRED && expected != SetupPhase.EXTERNAL_APPLY_REQUIRED) { + throw new SetupWorkflowConflict(); + } + state.ensurePhase(expected); + command.validate(validator); + ConfigurationResponse response = command.configure(configuration, capability); + state.configurationApplied(expected, response.operationId(), response.phase()); + return response; + } + + public String createAdministrator(AdministratorCommand command) { + try (command) { + requireWritable(); + state.ensurePhase(SetupPhase.ADMINISTRATOR_REQUIRED); + char[] clear = command.password().copy(); + try (AdministratorCredentials credentials = new AdministratorCredentials(command.username(), clear)) { + identities.orElseThrow(SetupWorkflowConflict::new).createFirstAdministrator(credentials); + } catch (BootstrapIdentityConflict conflict) { + throw new SetupApiException(SetupErrorCode.ADMINISTRATOR_ALREADY_CONFIGURED, HttpStatus.CONFLICT); + } finally { + Arrays.fill(clear, '\0'); + } + state.administratorCreated(command.username()); + return command.username(); + } + } + + public String complete(CompletionCommand command) { + requireWritable(); + state.ensurePhase(SetupPhase.OPTIONAL_CONFIGURATION); + if (command.expectedPhase() != SetupPhase.OPTIONAL_CONFIGURATION) { + throw new SetupWorkflowConflict(); + } + if (!command.acknowledgedWarnings().containsAll(state.pendingWarnings())) { + throw new SetupApiException(SetupErrorCode.OPERATION_CONFLICT, HttpStatus.CONFLICT); + } + String username = state.administratorUsername(); + if (username == null) { + throw new SetupWorkflowConflict(); + } + completion.orElseThrow(SetupWorkflowConflict::new).completeInstallation(); + state.complete(); + return username; + } + + private void requireWritable() { + if (state.phase() == SetupPhase.COMPLETE) { + throw new SetupApiException(SetupErrorCode.SETUP_COMPLETE, HttpStatus.GONE); + } + } + + /** Transport adapter for required configuration while the transition stays transport-neutral. */ + public interface ConfigurationCommand { + SetupPhase expectedPhase(); + + void validate(SetupRequestValidator validator); + + ConfigurationResponse configure(SetupConfigurationCoordinator coordinator, + ManagedConfigCapability capability); + + static ConfigurationCommand browser(ConfigurationRequest request) { + return new BrowserConfigurationCommand(request); + } + + static ConfigurationCommand headless(HeadlessSetupWorkflow.RequiredConfiguration request) { + return new HeadlessConfigurationCommand(request); + } + } + + /** Transition-owned administrator secret. */ + public record AdministratorCommand(String username, SecretValue password) implements AutoCloseable { + public static AdministratorCommand browser(AdministratorRequest request) { + return new AdministratorCommand(request.username(), SecretValue.of(request.password())); + } + + public static AdministratorCommand headless(String username, SecretValue password) { + return new AdministratorCommand(username, SecretValue.copyOf(password)); + } + + @Override + public void close() { + password.close(); + } + } + + /** Shared completion preconditions for every setup transport. */ + public record CompletionCommand(SetupPhase expectedPhase, List acknowledgedWarnings) { + public static CompletionCommand browser(CompleteRequest request) { + return new CompletionCommand(request.expectedPhase(), request.acknowledgedWarnings()); + } + + public static CompletionCommand headless(List acknowledgedWarnings) { + return new CompletionCommand(SetupPhase.OPTIONAL_CONFIGURATION, acknowledgedWarnings); + } + } + + private record BrowserConfigurationCommand(ConfigurationRequest request) implements ConfigurationCommand { + @Override + public SetupPhase expectedPhase() { + return request.expectedPhase(); + } + + @Override + public void validate(SetupRequestValidator validator) { + requireValid(validator, new ValidateRequest(ValidationSection.METADATA_DATABASE, + request.managementDatabase(), null, null, null)); + requireValid(validator, new ValidateRequest(ValidationSection.TELEMETRY_STORE, + null, request.telemetryStore(), null, null)); + } + + @Override + public ConfigurationResponse configure(SetupConfigurationCoordinator coordinator, + ManagedConfigCapability capability) { + return coordinator.configure(request, capability); + } + } + + private record HeadlessConfigurationCommand(HeadlessSetupWorkflow.RequiredConfiguration request) + implements ConfigurationCommand { + @Override + public SetupPhase expectedPhase() { + return request.expectedPhase(); + } + + @Override + public void validate(SetupRequestValidator validator) { + validator.validate(request.metadata()); + validator.validate(request.telemetry()); + } + + @Override + public ConfigurationResponse configure(SetupConfigurationCoordinator coordinator, + ManagedConfigCapability capability) { + return coordinator.configure(request, SetupConfigurationMapper.map(request), capability); + } + } + + private static void requireValid(SetupRequestValidator validator, ValidateRequest request) { + var response = validator.validate(request); + if (!response.valid()) { + throw new SetupApiException(response.errorCode(), HttpStatus.BAD_REQUEST); + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java index ca48e9b2bc..d28729da9e 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java @@ -180,7 +180,8 @@ class SetupApiContractTest { @Test void freezesStableSafeErrorCodes() throws Exception { assertWireValues(SetupErrorCode.values(), "setup_complete", "setup_locked", "setup_code_invalid", - "setup_code_expired", "setup_rate_limited", "setup_not_complete", "config_read_only", "config_write_failed", + "setup_code_expired", "setup_rate_limited", "setup_not_complete", "invalid_request", "internal_error", + "config_read_only", "config_write_failed", "config_recovery_required", "metadata_connection_failed", "metadata_kind_unsupported", "metadata_schema_mismatch", "metadata_insufficient_privileges", "telemetry_connection_failed", "public_address_invalid", "mail_connection_failed", "administrator_already_configured", diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupControllerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupControllerTest.java index 9295207f89..45ecaa29d2 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupControllerTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupControllerTest.java @@ -37,6 +37,7 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ManagementDatabas import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionalConfigurationSummary; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupAccess; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.StatusResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreKind; @@ -49,6 +50,7 @@ import org.apache.hertzbeat.manager.setup.workflow.SetupExportRenderer; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.http.MediaType; +import org.springframework.http.HttpStatus; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.setup.MockMvcBuilders; @@ -116,7 +118,7 @@ class SetupControllerTest { mvc.perform(post(SetupApiContract.UNLOCK_PATH).contentType(MediaType.APPLICATION_JSON).content("{}")) .andExpect(status().isBadRequest()) .andExpect(header().string("Cache-Control", "no-store")) - .andExpect(jsonPath("$.errorCode").value("setup_code_invalid")); + .andExpect(jsonPath("$.errorCode").value("invalid_request")); } @Test @@ -151,8 +153,18 @@ class SetupControllerTest { mvc.perform(get(SetupApiContract.STATUS_PATH)) .andExpect(status().isInternalServerError()) .andExpect(header().string("Cache-Control", "no-store")) - .andExpect(jsonPath("$.errorCode").value("config_write_failed")) + .andExpect(jsonPath("$.errorCode").value("internal_error")) .andExpect(content().string(org.hamcrest.Matchers.not( org.hamcrest.Matchers.containsString("database-secret")))); } + + @Test + void typedSetupFailureKeepsItsDomainErrorCode() throws Exception { + when(workflow.status()).thenThrow( + new SetupApiException(SetupErrorCode.CONFIG_READ_ONLY, HttpStatus.CONFLICT)); + + mvc.perform(get(SetupApiContract.STATUS_PATH)) + .andExpect(status().isConflict()) + .andExpect(jsonPath("$.errorCode").value("config_read_only")); + } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java index fb5dac4b94..3540ce9b82 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java @@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; import java.io.IOException; @@ -30,6 +31,8 @@ import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; @@ -61,12 +64,17 @@ class ManagedConfigurationTransactionTest { FileManagedApplicationConfigStore applicationStore = new FileManagedApplicationConfigStore(installationRoot); FileManagedSecretStore failingSecrets = new FileManagedSecretStore( installationRoot, new FailingOnceActivePublicationPublisher(new NioManagedFilePublisher())); + List diagnostics = new ArrayList<>(); ManagedConfigurationTransaction transaction = new ManagedConfigurationTransaction( - applicationStore, failingSecrets, installationRoot); + applicationStore, failingSecrets, installationRoot, diagnostics::add); assertEquals(ManagedConfigurationTransaction.Outcome.ROLLED_BACK, transaction.apply(bundle("next"))); assertActivePair("previous"); + assertThat(diagnostics).containsExactly(new RecoveryFailureReporter.Failure( + RecoveryFailureReporter.Stage.PROMOTE_CANDIDATE, + RecoveryFailureReporter.Store.SECRET, IOException.class.getName(), + RecoveryFailureReporter.SAFE_MESSAGE)); } @Test @@ -188,6 +196,30 @@ class ManagedConfigurationTransactionTest { assertThat(decoded.metadataDatabasePassword().copy()).containsOnly('\0'); } + @Test + void recoveryReportsSecretFreeStructuredDiagnosticsWithoutChangingTheWireOutcome() throws Exception { + ManagedApplicationConfigStore applications = mock(ManagedApplicationConfigStore.class); + ManagedSecretStore secretStore = mock(ManagedSecretStore.class); + when(applications.readActive()).thenReturn(CandidateRead.valid(configuration("owned"), "generation")); + when(applications.readCandidate()).thenReturn(CandidateRead.missing()); + when(applications.readLastKnownGood()).thenReturn(CandidateRead.missing()); + when(secretStore.readActive()).thenReturn(CandidateRead.valid(secrets("owned"), "generation")); + when(secretStore.readCandidate()).thenReturn(CandidateRead.missing()); + when(secretStore.readLastKnownGood()).thenReturn(CandidateRead.missing()); + doThrow(new IOException("must-not-be-reported")).when(applications).discardCandidate(); + List diagnostics = new ArrayList<>(); + + assertEquals(ManagedConfigurationTransaction.Outcome.RECOVERY_REQUIRED, + new ManagedConfigurationTransaction( + applications, secretStore, installationRoot, diagnostics::add).recover()); + + assertThat(diagnostics).containsExactly(new RecoveryFailureReporter.Failure( + RecoveryFailureReporter.Stage.DISCARD_CANDIDATE, + RecoveryFailureReporter.Store.APPLICATION, IOException.class.getName(), + RecoveryFailureReporter.SAFE_MESSAGE)); + assertThat(diagnostics.getFirst().toString()).doesNotContain("must-not-be-reported"); + } + private static void waitForFile(Path ready) throws Exception { long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); while (!Files.exists(ready) && System.nanoTime() < deadline) { diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java index de86398a95..c7daabd8fe 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java @@ -70,7 +70,7 @@ class DefaultSetupWorkflowTest { IdentityInitializationService identities = mock(IdentityInitializationService.class); SetupCompletionCoordinator completion = mock(SetupCompletionCoordinator.class); SetupMutationSerializer mutations = new SetupMutationSerializer(); - DefaultSetupWorkflow workflow = new DefaultSetupWorkflow(state, mock(SetupRequestValidator.class), + DefaultSetupWorkflow workflow = workflow(state, mock(SetupRequestValidator.class), mock(SetupConfigurationCoordinator.class), mock(SetupOperationRegistry.class), capability, Optional.of(identities), Optional.of(completion), mock(SetupOptionsCoordinator.class), Clock.systemUTC(), mutations); @@ -90,7 +90,7 @@ class DefaultSetupWorkflowTest { SetupPhase.OPTIONAL_CONFIGURATION, SetupAccess.LOCAL, true, "operator"); SetupCompletionCoordinator completion = mock(SetupCompletionCoordinator.class); SetupMutationSerializer mutations = new SetupMutationSerializer(); - DefaultSetupWorkflow workflow = new DefaultSetupWorkflow(state, mock(SetupRequestValidator.class), + DefaultSetupWorkflow workflow = workflow(state, mock(SetupRequestValidator.class), mock(SetupConfigurationCoordinator.class), mock(SetupOperationRegistry.class), capability, Optional.of(mock(IdentityInitializationService.class)), Optional.of(completion), mock(SetupOptionsCoordinator.class), Clock.systemUTC(), mutations); @@ -122,11 +122,11 @@ class DefaultSetupWorkflowTest { }).when(options).persist(any()); SetupCompletionCoordinator completion = mock(SetupCompletionCoordinator.class); SetupMutationSerializer mutations = new SetupMutationSerializer(); - DefaultSetupWorkflow workflow = new DefaultSetupWorkflow(state, validator, + DefaultSetupWorkflow workflow = workflow(state, validator, mock(SetupConfigurationCoordinator.class), mock(SetupOperationRegistry.class), capability, Optional.of(mock(IdentityInitializationService.class)), Optional.of(completion), options, Clock.systemUTC(), mutations); - HeadlessSetupCoordinator headless = new HeadlessSetupCoordinator(state, validator, + HeadlessSetupCoordinator headless = headless(state, validator, mock(SetupConfigurationCoordinator.class), capability, Optional.of(mock(IdentityInitializationService.class)), Optional.of(completion), mutations); OptionsRequest request = new OptionsRequest( @@ -167,12 +167,12 @@ class DefaultSetupWorkflowTest { SetupPhase.APPLICATION_STARTING, 0, false); }).when(configuration).configure(any(ConfigurationRequest.class), eq(capability)); SetupMutationSerializer mutations = new SetupMutationSerializer(); - DefaultSetupWorkflow browser = new DefaultSetupWorkflow(state, validator, configuration, + DefaultSetupWorkflow browser = workflow(state, validator, configuration, mock(SetupOperationRegistry.class), capability, Optional.of(mock(IdentityInitializationService.class)), Optional.of(mock(SetupCompletionCoordinator.class)), mock(SetupOptionsCoordinator.class), Clock.systemUTC(), mutations); - HeadlessSetupCoordinator headless = new HeadlessSetupCoordinator(state, validator, configuration, + HeadlessSetupCoordinator headless = headless(state, validator, configuration, capability, Optional.of(mock(IdentityInitializationService.class)), Optional.of(mock(SetupCompletionCoordinator.class)), mutations); ConfigurationRequest browserRequest = new ConfigurationRequest( @@ -187,7 +187,8 @@ class DefaultSetupWorkflowTest { var browserResult = executor.submit(() -> browser.configure(browserRequest)); configurationStarted.await(5, TimeUnit.SECONDS); var headlessResult = executor.submit(() -> headless.configure( - new HeadlessSetupWorkflow.RequiredConfiguration(ApplyMode.MANAGED_WRITE, + new HeadlessSetupWorkflow.RequiredConfiguration(SetupPhase.CONFIGURATION_REQUIRED, + ApplyMode.MANAGED_WRITE, new HeadlessSetupWorkflow.Metadata(MetadataDatabaseKind.H2, "jdbc:h2:mem:headless", "sa", metadataPassword), new HeadlessSetupWorkflow.Telemetry("localhost:4001", @@ -204,4 +205,26 @@ class DefaultSetupWorkflowTest { verify(configuration, never()).configure(any(HeadlessSetupWorkflow.RequiredConfiguration.class), any(ManagedConfigurationBundle.class), eq(capability)); } + + private static DefaultSetupWorkflow workflow( + SetupRuntimeState state, SetupRequestValidator validator, + SetupConfigurationCoordinator configuration, SetupOperationRegistry operations, + ManagedConfigCapability capability, Optional identities, + Optional completion, SetupOptionsCoordinator options, + Clock clock, SetupMutationSerializer mutations) { + SetupTransitionService transitions = new SetupTransitionService( + state, validator, configuration, capability, identities, completion); + return new DefaultSetupWorkflow(state, validator, operations, + options, clock, mutations, transitions); + } + + private static HeadlessSetupCoordinator headless( + SetupRuntimeState state, SetupRequestValidator validator, + SetupConfigurationCoordinator configuration, ManagedConfigCapability capability, + Optional identities, + Optional completion, SetupMutationSerializer mutations) { + SetupTransitionService transitions = new SetupTransitionService( + state, validator, configuration, capability, identities, completion); + return new HeadlessSetupCoordinator(state, mutations, transitions); + } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinatorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinatorTest.java index 92fd844800..92a3863c22 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinatorTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinatorTest.java @@ -44,10 +44,11 @@ class HeadlessSetupCoordinatorTest { List.of(SetupWarningCode.H2_NON_PRODUCTION, SetupWarningCode.PUBLIC_ADDRESS_PLAINTEXT)); SetupCompletionCoordinator completion = mock(SetupCompletionCoordinator.class); - HeadlessSetupCoordinator coordinator = new HeadlessSetupCoordinator(state, + SetupTransitionService transitions = new SetupTransitionService(state, mock(SetupRequestValidator.class), mock(SetupConfigurationCoordinator.class), capability, - Optional.of(mock(IdentityInitializationService.class)), Optional.of(completion), - new SetupMutationSerializer()); + Optional.of(mock(IdentityInitializationService.class)), Optional.of(completion)); + HeadlessSetupCoordinator coordinator = new HeadlessSetupCoordinator( + state, new SetupMutationSerializer(), transitions); assertThrows(SetupApiException.class, () -> coordinator.complete(List.of(SetupWarningCode.H2_NON_PRODUCTION))); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationCoordinatorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationCoordinatorTest.java index a145334e59..69fbf9e926 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationCoordinatorTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationCoordinatorTest.java @@ -20,6 +20,7 @@ package org.apache.hertzbeat.manager.setup.workflow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -85,13 +86,35 @@ class SetupConfigurationCoordinatorTest { () -> coordinator.configure(request(ApplyMode.EXTERNAL_APPLY), capability)); } + @Test + void externalApplyReentryReturnsFreshAcknowledgementWithoutPersistingSubmittedSecrets() { + SetupOperationRegistry operations = new SetupOperationRegistry(Clock.systemUTC()); + SetupConfigurationCoordinator coordinator = new SetupConfigurationCoordinator( + new ManagedConfigurationTransaction(installationRoot), operations); + ManagedConfigCapability capability = new ManagedConfigCapability( + ApplyMode.EXTERNAL_APPLY, false, DeploymentConstraint.READ_ONLY); + + var first = coordinator.configure(request(SetupPhase.CONFIGURATION_REQUIRED, ApplyMode.EXTERNAL_APPLY), + capability); + var replacement = coordinator.configure( + request(SetupPhase.EXTERNAL_APPLY_REQUIRED, ApplyMode.EXTERNAL_APPLY), capability); + + assertNotEquals(first.operationId(), replacement.operationId()); + assertEquals(SetupOperationState.AWAITING_EXTERNAL_APPLY, replacement.state()); + assertEquals(SetupPhase.EXTERNAL_APPLY_REQUIRED, replacement.phase()); + assertTrue(replacement.exportAvailable()); + assertEquals(ManagedActiveConfigurationInspector.State.ABSENT, + new ManagedActiveConfigurationInspector(installationRoot).inspect().state()); + } + @Test void headlessExternalApplyClosesTheCoordinatorOwnedSecretBundle() { SetupConfigurationCoordinator coordinator = new SetupConfigurationCoordinator( new ManagedConfigurationTransaction(installationRoot), new SetupOperationRegistry(Clock.systemUTC())); try (SecretValue callerPassword = SecretValue.of("metadata-password")) { - var request = new HeadlessSetupWorkflow.RequiredConfiguration(ApplyMode.EXTERNAL_APPLY, + var request = new HeadlessSetupWorkflow.RequiredConfiguration(SetupPhase.CONFIGURATION_REQUIRED, + ApplyMode.EXTERNAL_APPLY, new HeadlessSetupWorkflow.Metadata(MetadataDatabaseKind.H2, "jdbc:h2:./data/setup", "sa", callerPassword), new HeadlessSetupWorkflow.Telemetry("localhost:4001", "http://localhost:4000", @@ -114,7 +137,11 @@ class SetupConfigurationCoordinatorTest { } private static ConfigurationRequest request(ApplyMode applyMode) { - return new ConfigurationRequest(SetupPhase.CONFIGURATION_REQUIRED, applyMode, + return request(SetupPhase.CONFIGURATION_REQUIRED, applyMode); + } + + private static ConfigurationRequest request(SetupPhase expectedPhase, ApplyMode applyMode) { + return new ConfigurationRequest(expectedPhase, applyMode, new MetadataDatabaseConfiguration(MetadataDatabaseKind.H2, "jdbc:h2:./data/setup", "sa", "metadata-password"), new TelemetryStoreConfiguration(TelemetryStoreKind.GREPTIME, diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationMapperTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationMapperTest.java index 92bbf0d153..9315ba79ae 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationMapperTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationMapperTest.java @@ -22,6 +22,7 @@ import static org.assertj.core.api.Assertions.assertThat; import java.util.Optional; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; import org.apache.hertzbeat.manager.setup.config.SecretValue; import org.junit.jupiter.api.Test; @@ -30,7 +31,8 @@ class SetupConfigurationMapperTest { void headlessMappingCopiesCallerOwnedSecretsIntoCoordinatorOwnedBundle() { SecretValue metadata = SecretValue.of("metadata-secret"); SecretValue telemetry = SecretValue.of("telemetry-secret"); - var request = new HeadlessSetupWorkflow.RequiredConfiguration(ApplyMode.MANAGED_WRITE, + var request = new HeadlessSetupWorkflow.RequiredConfiguration(SetupPhase.CONFIGURATION_REQUIRED, + ApplyMode.MANAGED_WRITE, new HeadlessSetupWorkflow.Metadata(MetadataDatabaseKind.H2, "jdbc:h2:./data/setup", "sa", metadata), new HeadlessSetupWorkflow.Telemetry("localhost:4001", "http://localhost:4000", diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionServiceTest.java new file mode 100644 index 0000000000..2972781367 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionServiceTest.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.List; +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.AdministratorRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.CompleteRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupAccess; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiException; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.apache.hertzbeat.manager.setup.identity.IdentityInitializationService; +import org.junit.jupiter.api.Test; + +class SetupTransitionServiceTest { + private static final Clock CLOCK = Clock.fixed(Instant.parse("2026-08-08T00:00:00Z"), ZoneOffset.UTC); + + @Test + void browserAndHeadlessConfigurationUseTheSamePhaseValidationAndStateTransition() { + ManagedConfigCapability capability = mock(ManagedConfigCapability.class); + SetupRuntimeState state = state(capability, SetupPhase.CONFIGURATION_REQUIRED, false, null); + SetupRequestValidator validator = mock(SetupRequestValidator.class); + when(validator.validate(any(org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidateRequest.class))) + .thenReturn(new ValidationResponse(true, CLOCK.instant(), null, List.of())); + SetupConfigurationCoordinator configuration = mock(SetupConfigurationCoordinator.class); + when(configuration.configure(any(ConfigurationRequest.class), any())) + .thenReturn(configurationResponse("browser")); + when(configuration.configure(any(HeadlessSetupWorkflow.RequiredConfiguration.class), any(), any())) + .thenReturn(configurationResponse("headless")); + SetupTransitionService transitions = transitions(state, validator, configuration, capability, + Optional.empty(), Optional.empty()); + + transitions.configure(SetupTransitionService.ConfigurationCommand.browser(browserConfiguration())); + assertThat(state.phase()).isEqualTo(SetupPhase.APPLICATION_STARTING); + + SetupRuntimeState headlessState = state(capability, SetupPhase.CONFIGURATION_REQUIRED, false, null); + SetupTransitionService headlessTransitions = transitions(headlessState, validator, configuration, capability, + Optional.empty(), Optional.empty()); + try (SecretValue metadataPassword = SecretValue.of("secret")) { + headlessTransitions.configure(SetupTransitionService.ConfigurationCommand.headless( + headlessConfiguration(metadataPassword))); + } + assertThat(headlessState.phase()).isEqualTo(SetupPhase.APPLICATION_STARTING); + } + + @Test + void browserAndHeadlessAdministratorCommandsRejectTheSameWrongPhaseBeforeSideEffects() { + ManagedConfigCapability capability = mock(ManagedConfigCapability.class); + IdentityInitializationService identities = mock(IdentityInitializationService.class); + + for (boolean browser : List.of(true, false)) { + SetupRuntimeState state = state(capability, SetupPhase.CONFIGURATION_REQUIRED, false, null); + SetupTransitionService transitions = transitions(state, mock(SetupRequestValidator.class), + mock(SetupConfigurationCoordinator.class), capability, Optional.of(identities), Optional.empty()); + assertThatThrownBy(() -> { + if (browser) { + transitions.createAdministrator(SetupTransitionService.AdministratorCommand.browser( + new AdministratorRequest("operator", "secret"))); + } else { + try (SecretValue password = SecretValue.of("secret")) { + transitions.createAdministrator( + SetupTransitionService.AdministratorCommand.headless("operator", password)); + } + } + }).isInstanceOf(SetupWorkflowConflict.class); + } + verifyNoInteractions(identities); + } + + @Test + void externalApplyReentryIsExplicitAndStillRunsValidationForBothTransports() { + ManagedConfigCapability capability = mock(ManagedConfigCapability.class); + SetupRequestValidator validator = mock(SetupRequestValidator.class); + when(validator.validate(any(org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidateRequest.class))) + .thenReturn(new ValidationResponse(true, CLOCK.instant(), null, List.of())); + SetupConfigurationCoordinator configuration = mock(SetupConfigurationCoordinator.class); + ConfigurationResponse response = new ConfigurationResponse("replacement", + SetupOperationState.AWAITING_EXTERNAL_APPLY, SetupPhase.EXTERNAL_APPLY_REQUIRED, 0, true); + when(configuration.configure(any(ConfigurationRequest.class), any())).thenReturn(response); + when(configuration.configure(any(HeadlessSetupWorkflow.RequiredConfiguration.class), any(), any())) + .thenReturn(response); + + SetupRuntimeState browserState = state(capability, SetupPhase.EXTERNAL_APPLY_REQUIRED, false, null); + SetupTransitionService browser = transitions(browserState, validator, configuration, capability, + Optional.empty(), Optional.empty()); + browser.configure(SetupTransitionService.ConfigurationCommand.browser( + browserConfiguration(SetupPhase.EXTERNAL_APPLY_REQUIRED, ApplyMode.EXTERNAL_APPLY))); + + SetupRuntimeState headlessState = state(capability, SetupPhase.EXTERNAL_APPLY_REQUIRED, false, null); + SetupTransitionService headless = transitions(headlessState, validator, configuration, capability, + Optional.empty(), Optional.empty()); + try (SecretValue password = SecretValue.of("secret")) { + headless.configure(SetupTransitionService.ConfigurationCommand.headless( + headlessConfiguration(SetupPhase.EXTERNAL_APPLY_REQUIRED, ApplyMode.EXTERNAL_APPLY, password))); + } + + assertThat(browserState.phase()).isEqualTo(SetupPhase.EXTERNAL_APPLY_REQUIRED); + assertThat(headlessState.phase()).isEqualTo(SetupPhase.EXTERNAL_APPLY_REQUIRED); + verify(configuration).configure(any(ConfigurationRequest.class), any()); + verify(configuration).configure(any(HeadlessSetupWorkflow.RequiredConfiguration.class), any(), any()); + } + + @Test + void configurationReentryNeverExtendsToApplicationStarting() { + ManagedConfigCapability capability = mock(ManagedConfigCapability.class); + SetupRequestValidator validator = mock(SetupRequestValidator.class); + SetupConfigurationCoordinator configuration = mock(SetupConfigurationCoordinator.class); + SetupTransitionService transitions = transitions( + state(capability, SetupPhase.APPLICATION_STARTING, false, null), validator, configuration, + capability, Optional.empty(), Optional.empty()); + + assertThatThrownBy(() -> transitions.configure(SetupTransitionService.ConfigurationCommand.browser( + browserConfiguration(SetupPhase.APPLICATION_STARTING, ApplyMode.EXTERNAL_APPLY)))) + .isInstanceOf(SetupWorkflowConflict.class); + verifyNoInteractions(validator, configuration); + } + + @Test + void browserAndHeadlessCompletionUseTheSameWarningGateAndCompletionSideEffect() { + ManagedConfigCapability capability = mock(ManagedConfigCapability.class); + SetupCompletionCoordinator completion = mock(SetupCompletionCoordinator.class); + + for (boolean browser : List.of(true, false)) { + SetupRuntimeState state = state(capability, SetupPhase.OPTIONAL_CONFIGURATION, true, "operator"); + SetupTransitionService transitions = transitions(state, mock(SetupRequestValidator.class), + mock(SetupConfigurationCoordinator.class), capability, Optional.empty(), Optional.of(completion)); + SetupTransitionService.CompletionCommand command = browser + ? SetupTransitionService.CompletionCommand.browser( + new CompleteRequest(SetupPhase.OPTIONAL_CONFIGURATION, List.of())) + : SetupTransitionService.CompletionCommand.headless(List.of()); + + assertThatThrownBy(() -> transitions.complete(command)) + .isInstanceOfSatisfying(SetupApiException.class, + failure -> assertThat(failure.errorCode()).isEqualTo(SetupErrorCode.OPERATION_CONFLICT)); + assertThat(state.phase()).isEqualTo(SetupPhase.OPTIONAL_CONFIGURATION); + } + verify(completion, never()).completeInstallation(); + } + + private static SetupTransitionService transitions( + SetupRuntimeState state, SetupRequestValidator validator, + SetupConfigurationCoordinator configuration, ManagedConfigCapability capability, + Optional identities, Optional completion) { + return new SetupTransitionService(state, validator, configuration, capability, identities, completion); + } + + private static SetupRuntimeState state(ManagedConfigCapability capability, SetupPhase phase, + boolean administratorConfigured, String username) { + return new SetupRuntimeState(CLOCK, capability, phase, SetupAccess.LOCAL, + administratorConfigured, username); + } + + private static ConfigurationResponse configurationResponse(String id) { + return new ConfigurationResponse(id, SetupOperationState.AWAITING_RESTART, + SetupPhase.APPLICATION_STARTING, 0, false); + } + + private static ConfigurationRequest browserConfiguration() { + return browserConfiguration(SetupPhase.CONFIGURATION_REQUIRED, ApplyMode.MANAGED_WRITE); + } + + private static ConfigurationRequest browserConfiguration(SetupPhase expectedPhase, ApplyMode applyMode) { + return new ConfigurationRequest(expectedPhase, applyMode, + new MetadataDatabaseConfiguration(MetadataDatabaseKind.H2, "jdbc:h2:mem:browser", "sa", "secret"), + new TelemetryStoreConfiguration(TelemetryStoreKind.GREPTIME, + "localhost:4001", "http://localhost:4000", "public", null, null)); + } + + private static HeadlessSetupWorkflow.RequiredConfiguration headlessConfiguration(SecretValue password) { + return headlessConfiguration(SetupPhase.CONFIGURATION_REQUIRED, ApplyMode.MANAGED_WRITE, password); + } + + private static HeadlessSetupWorkflow.RequiredConfiguration headlessConfiguration( + SetupPhase expectedPhase, ApplyMode applyMode, SecretValue password) { + return new HeadlessSetupWorkflow.RequiredConfiguration(expectedPhase, applyMode, + new HeadlessSetupWorkflow.Metadata(MetadataDatabaseKind.H2, "jdbc:h2:mem:headless", "sa", password), + new HeadlessSetupWorkflow.Telemetry("localhost:4001", "http://localhost:4000", "public", + Optional.empty(), Optional.empty())); + } +} From 4f6fd62a6c397196c92a27f21f9d9bd0b9e9a036 Mon Sep 17 00:00:00 2001 From: Logic Date: Sat, 8 Aug 2026 23:00:15 +0800 Subject: [PATCH 13/71] Normalize setup administrator identity --- .../manager/setup/api/SetupApiContract.java | 2 +- .../identity/AdministratorCredentials.java | 6 +-- .../setup/identity/DatabaseAccount.java | 3 +- .../IdentityInitializationService.java | 4 +- .../InvalidAdministratorUsername.java | 12 +++++ .../workflow/SetupTransitionService.java | 9 +++- .../setup/api/SetupControllerTest.java | 17 ++++++ .../IdentityInitializationServiceTest.java | 16 ++++++ .../workflow/SetupTransitionServiceTest.java | 53 +++++++++++++++++++ 9 files changed, 113 insertions(+), 9 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/InvalidAdministratorUsername.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java index c1784e9386..2e72b4e7b7 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java @@ -504,7 +504,7 @@ public final class SetupApiContract { /** Initial administrator input. */ public record AdministratorRequest( - @NotBlank String username, + String username, @NotBlank @JsonProperty(access = JsonProperty.Access.WRITE_ONLY) String password) { @Override diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/AdministratorCredentials.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/AdministratorCredentials.java index c84acf5946..38cddf5284 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/AdministratorCredentials.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/AdministratorCredentials.java @@ -27,8 +27,8 @@ public final class AdministratorCredentials implements AutoCloseable { public AdministratorCredentials(String username, char[] password) { String normalizedUsername = StringUtils.trimToNull(username); - if (normalizedUsername == null) { - throw new IllegalArgumentException("Administrator username is required"); + if (normalizedUsername == null || normalizedUsername.length() > DatabaseAccount.USERNAME_MAX_LENGTH) { + throw new InvalidAdministratorUsername(); } if (password == null || password.length == 0) { throw new IllegalArgumentException("Administrator password is required"); @@ -37,7 +37,7 @@ public final class AdministratorCredentials implements AutoCloseable { this.password = password.clone(); } - String username() { + public String canonicalUsername() { return username; } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/DatabaseAccount.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/DatabaseAccount.java index fbe9c2f5e1..cf537942cd 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/DatabaseAccount.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/DatabaseAccount.java @@ -34,12 +34,13 @@ import org.apache.commons.lang3.StringUtils; @UniqueConstraint(name = "uk_hzb_account_username", columnNames = "username"), @UniqueConstraint(name = "uk_hzb_account_bootstrap", columnNames = "bootstrap_slot")}) public class DatabaseAccount { + static final int USERNAME_MAX_LENGTH = 64; @Id @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; - @Column(nullable = false, length = 64) + @Column(nullable = false, length = USERNAME_MAX_LENGTH) private String username; @Column(name = "password_hash", nullable = false, length = 100) diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/IdentityInitializationService.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/IdentityInitializationService.java index 27b8001852..a2c7e10174 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/IdentityInitializationService.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/IdentityInitializationService.java @@ -41,12 +41,12 @@ public class IdentityInitializationService { char[] clear = credentials.copyPassword(); try { if (accounts.existsByBootstrapSlotIsNotNull() - || accounts.existsByUsername(credentials.username())) { + || accounts.existsByUsername(credentials.canonicalUsername())) { throw new BootstrapIdentityConflict(); } try { accounts.saveAndFlush(DatabaseAccount.firstAdministrator( - credentials.username(), passwords.encode(clear), "admin")); + credentials.canonicalUsername(), passwords.encode(clear), "admin")); } catch (DataIntegrityViolationException exception) { throw BootstrapIdentityConflict.map(exception); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/InvalidAdministratorUsername.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/InvalidAdministratorUsername.java new file mode 100644 index 0000000000..a32ef30ed1 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/identity/InvalidAdministratorUsername.java @@ -0,0 +1,12 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.identity; + +/** Domain signal for an administrator username that cannot be persisted canonically. */ +public final class InvalidAdministratorUsername extends IllegalArgumentException { +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionService.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionService.java index 154e6a8e3c..83eebf013b 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionService.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionService.java @@ -25,6 +25,7 @@ import org.apache.hertzbeat.manager.setup.config.SecretValue; import org.apache.hertzbeat.manager.setup.identity.AdministratorCredentials; import org.apache.hertzbeat.manager.setup.identity.BootstrapIdentityConflict; import org.apache.hertzbeat.manager.setup.identity.IdentityInitializationService; +import org.apache.hertzbeat.manager.setup.identity.InvalidAdministratorUsername; import org.springframework.http.HttpStatus; /** Single state transition boundary shared by browser and headless setup adapters. */ @@ -66,15 +67,19 @@ public final class SetupTransitionService { requireWritable(); state.ensurePhase(SetupPhase.ADMINISTRATOR_REQUIRED); char[] clear = command.password().copy(); + String canonicalUsername; try (AdministratorCredentials credentials = new AdministratorCredentials(command.username(), clear)) { + canonicalUsername = credentials.canonicalUsername(); identities.orElseThrow(SetupWorkflowConflict::new).createFirstAdministrator(credentials); + } catch (InvalidAdministratorUsername invalid) { + throw new SetupApiException(SetupErrorCode.ADMINISTRATOR_USERNAME_INVALID, HttpStatus.BAD_REQUEST); } catch (BootstrapIdentityConflict conflict) { throw new SetupApiException(SetupErrorCode.ADMINISTRATOR_ALREADY_CONFIGURED, HttpStatus.CONFLICT); } finally { Arrays.fill(clear, '\0'); } - state.administratorCreated(command.username()); - return command.username(); + state.administratorCreated(canonicalUsername); + return canonicalUsername; } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupControllerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupControllerTest.java index 45ecaa29d2..69eee3b206 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupControllerTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupControllerTest.java @@ -121,6 +121,23 @@ class SetupControllerTest { .andExpect(jsonPath("$.errorCode").value("invalid_request")); } + @Test + void administratorUsernameDomainErrorWinsOverTransportValidation() throws Exception { + when(workflow.createAdministrator(any())).thenThrow( + new SetupApiException(SetupErrorCode.ADMINISTRATOR_USERNAME_INVALID, HttpStatus.BAD_REQUEST)); + + for (String username : new String[] {"", " "}) { + String request = "{\"username\":\"" + username + "\",\"password\":\"request-secret\"}"; + mvc.perform(post(SetupApiContract.ADMINISTRATOR_PATH) + .contentType(MediaType.APPLICATION_JSON).content(request)) + .andExpect(status().isBadRequest()) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.errorCode").value("administrator_username_invalid")) + .andExpect(content().string(org.hamcrest.Matchers.not( + org.hamcrest.Matchers.containsString("request-secret")))); + } + } + @Test void exportIsActualNoStoreAttachmentRatherThanMetadataJson() throws Exception { when(workflow.prepareExport(any())).thenReturn( diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/IdentityInitializationServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/IdentityInitializationServiceTest.java index 23b4d22fd8..9462b157fd 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/IdentityInitializationServiceTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/identity/IdentityInitializationServiceTest.java @@ -34,6 +34,22 @@ import org.springframework.dao.DataIntegrityViolationException; import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; class IdentityInitializationServiceTest { + @Test + void administratorUsernameUsesCanonicalDatabaseBoundaryAndRejectsOverflow() { + try (AdministratorCredentials maximum = new AdministratorCredentials( + " xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx ", + "secret".toCharArray())) { + assertEquals("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + maximum.canonicalUsername()); + } + assertThrows(InvalidAdministratorUsername.class, + () -> new AdministratorCredentials(" ", "secret".toCharArray())); + assertThrows(InvalidAdministratorUsername.class, + () -> new AdministratorCredentials( + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "secret".toCharArray())); + } + @Test void createsUniqueFirstAdministratorWithCostTwelveHash() { DatabaseAccountRepository repository = mock(DatabaseAccountRepository.class); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionServiceTest.java index 2972781367..f9f47b7240 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionServiceTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionServiceTest.java @@ -12,6 +12,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; @@ -39,6 +40,7 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiException; import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; import org.apache.hertzbeat.manager.setup.config.SecretValue; import org.apache.hertzbeat.manager.setup.identity.IdentityInitializationService; +import org.apache.hertzbeat.manager.setup.identity.BootstrapIdentityConflict; import org.junit.jupiter.api.Test; class SetupTransitionServiceTest { @@ -96,6 +98,57 @@ class SetupTransitionServiceTest { verifyNoInteractions(identities); } + @Test + void administratorUsesOneCanonicalUsernameForIdentityRuntimeAndResponse() { + ManagedConfigCapability capability = mock(ManagedConfigCapability.class); + IdentityInitializationService identities = mock(IdentityInitializationService.class); + SetupRuntimeState state = state(capability, SetupPhase.ADMINISTRATOR_REQUIRED, false, null); + SetupTransitionService transitions = transitions(state, mock(SetupRequestValidator.class), + mock(SetupConfigurationCoordinator.class), capability, Optional.of(identities), Optional.empty()); + + String responseUsername = transitions.createAdministrator( + SetupTransitionService.AdministratorCommand.browser( + new AdministratorRequest(" operator ", "secret"))); + + assertThat(responseUsername).isEqualTo("operator"); + assertThat(state.administratorUsername()).isEqualTo("operator"); + verify(identities).createFirstAdministrator(any()); + } + + @Test + void invalidAdministratorUsernameHasStableCodeWhileUniqueConflictKeepsConflictCode() { + ManagedConfigCapability capability = mock(ManagedConfigCapability.class); + IdentityInitializationService identities = mock(IdentityInitializationService.class); + SetupRuntimeState state = state(capability, SetupPhase.ADMINISTRATOR_REQUIRED, false, null); + SetupTransitionService transitions = transitions(state, mock(SetupRequestValidator.class), + mock(SetupConfigurationCoordinator.class), capability, Optional.of(identities), Optional.empty()); + + for (String username : List.of(" ", "x".repeat(65))) { + SetupTransitionService.AdministratorCommand command = + SetupTransitionService.AdministratorCommand.browser( + new AdministratorRequest(username, "secret")); + assertThatThrownBy(() -> transitions.createAdministrator(command)) + .isInstanceOfSatisfying(SetupApiException.class, + failure -> assertThat(failure.errorCode()) + .isEqualTo(SetupErrorCode.ADMINISTRATOR_USERNAME_INVALID)); + assertThat(command.password().copy()).containsOnly('\0'); + } + verifyNoInteractions(identities); + + IdentityInitializationService conflicting = mock(IdentityInitializationService.class); + doThrow(new BootstrapIdentityConflict()).when(conflicting).createFirstAdministrator(any()); + SetupRuntimeState conflictState = state(capability, SetupPhase.ADMINISTRATOR_REQUIRED, false, null); + SetupTransitionService conflictTransitions = transitions( + conflictState, mock(SetupRequestValidator.class), mock(SetupConfigurationCoordinator.class), + capability, Optional.of(conflicting), Optional.empty()); + assertThatThrownBy(() -> conflictTransitions.createAdministrator( + SetupTransitionService.AdministratorCommand.browser( + new AdministratorRequest("operator", "secret")))) + .isInstanceOfSatisfying(SetupApiException.class, + failure -> assertThat(failure.errorCode()) + .isEqualTo(SetupErrorCode.ADMINISTRATOR_ALREADY_CONFIGURED)); + } + @Test void externalApplyReentryIsExplicitAndStillRunsValidationForBothTransports() { ManagedConfigCapability capability = mock(ManagedConfigCapability.class); From 57548d9d088ab3f1866477c023d1ee1d9493484b Mon Sep 17 00:00:00 2001 From: Logic Date: Sat, 8 Aug 2026 23:42:56 +0800 Subject: [PATCH 14/71] Keep setup reachable in gated runtime --- .../src/main/resources/sureness.yml | 2 + .../StartupRuntimeBoundaryContextTest.java | 59 +++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/hertzbeat-startup/src/main/resources/sureness.yml b/hertzbeat-startup/src/main/resources/sureness.yml index 415d5d4c23..f470a3b75e 100644 --- a/hertzbeat-startup/src/main/resources/sureness.yml +++ b/hertzbeat-startup/src/main/resources/sureness.yml @@ -165,6 +165,8 @@ resourceRole: # rule: api===method # eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth. excludedResource: + # Setup has its own runtime gate, remote-unlock proof, and permanent completion closure. + - /api/setup/**===* - /api/account/auth/**===* - /api/ui/session===get - /api/ui/session===post diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupRuntimeBoundaryContextTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupRuntimeBoundaryContextTest.java index db3abaf09f..8795bdb464 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupRuntimeBoundaryContextTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupRuntimeBoundaryContextTest.java @@ -22,11 +22,20 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Path; import org.apache.hertzbeat.common.runtime.BusinessRuntimeGate; import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; +import org.apache.hertzbeat.manager.setup.workflow.SetupRuntimeState; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.boot.web.server.context.WebServerApplicationContext; import org.springframework.context.ConfigurableApplicationContext; class StartupRuntimeBoundaryContextTest { @@ -47,6 +56,52 @@ class StartupRuntimeBoundaryContextTest { "questdbDataStorage" }; + @TempDir + Path installationRoot; + + @Test + void fullGatedSurenessChainKeepsSetupReachableAndBusinessRoutesClosed() throws Exception { + SpringStartupContextLauncher launcher = new SpringStartupContextLauncher(); + StartupDecision decision = new StartupDecision( + RuntimeMode.FULL_SETUP_GATED, SetupPhase.ADMINISTRATOR_REQUIRED, null); + String databaseName = "m5_setup_security_" + System.nanoTime(); + try (ConfigurableApplicationContext context = launcher.launchSpringContext(decision, new String[]{ + "--spring.profiles.active=test", + "--server.port=0", + "--spring.datasource.url=jdbc:h2:mem:" + databaseName + ";MODE=MYSQL;DB_CLOSE_DELAY=-1", + "--spring.flyway.enabled=false", + "--hertzbeat.installation.root=" + installationRoot, + "--warehouse.store.duckdb.enabled=false", + "--warehouse.store.greptime.enabled=false", + "--hertzbeat.runtime.mode=normal" + }, SETUP_RUNTIME_TRANSITION); + HttpClient client = HttpClient.newHttpClient()) { + int port = ((WebServerApplicationContext) context).getWebServer().getPort(); + + HttpResponse status = client.send( + request(port, "/api/setup/status").GET().build(), HttpResponse.BodyHandlers.ofString()); + assertEquals(200, status.statusCode()); + assertEquals("administrator_required", JsonUtil.fromJson(status.body()).path("phase").asText()); + + HttpResponse business = client.send( + request(port, "/api/monitors").GET().build(), HttpResponse.BodyHandlers.ofString()); + assertEquals(503, business.statusCode()); + assertEquals("setup_not_complete", JsonUtil.fromJson(business.body()).path("msg").asText()); + + SetupRuntimeState state = context.getBean(SetupRuntimeState.class); + state.administratorCreated("admin"); + state.complete(); + HttpResponse completedWrite = client.send(request(port, "/api/setup/administrator") + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString( + "{\"username\":\"admin\",\"password\":\"not-retained\"}")) + .build(), + HttpResponse.BodyHandlers.ofString()); + assertEquals(410, completedWrite.statusCode()); + assertEquals("setup_complete", JsonUtil.fromJson(completedWrite.body()).path("errorCode").asText()); + } + } + @Test void realFullApplicationStartsGatedWithoutBusinessSideEffectsOrCliBypass() { SpringStartupContextLauncher launcher = new SpringStartupContextLauncher(); @@ -112,4 +167,8 @@ class StartupRuntimeBoundaryContextTest { assertFalse(context.containsBeanDefinition("otlpGrpcServerConfig")); } } + + private static HttpRequest.Builder request(int port, String path) { + return HttpRequest.newBuilder(URI.create("http://127.0.0.1:" + port + path)); + } } From 801ff3ad02fd993fc45ffa968c9ae69fc1550876 Mon Sep 17 00:00:00 2001 From: Logic Date: Sun, 9 Aug 2026 01:19:26 +0800 Subject: [PATCH 15/71] Wire setup options to runtime consumers --- .../impl/EmailAlertNotifyHandlerImpl.java | 5 +- .../impl/EmailAlertNotifyHandlerImplTest.java | 31 ++++- hertzbeat-manager/pom.xml | 1 - .../manager/setup/api/SetupApiContract.java | 38 ++---- .../api/SetupStatusProjectionFactory.java | 56 ++++++--- .../ApplicationConfigDocumentCodec.java | 119 ++++++++++++------ .../config/ManagedConfigurationKeys.java | 15 +-- .../config/ManagedOptionalConfiguration.java | 44 ++++--- .../setup/workflow/DefaultSetupWorkflow.java | 23 ++-- .../PublicAccessConfigurationValidator.java | 75 ----------- ...InstrumentationConfigurationValidator.java | 59 +++++++++ .../SetupConfigurationProjection.java | 2 +- .../workflow/SetupOptionsCoordinator.java | 16 ++- .../setup/workflow/SetupRequestValidator.java | 7 +- .../setup/workflow/SetupWarningPolicy.java | 22 +++- .../setup/api/SetupApiContractTest.java | 25 ++-- .../setup/api/SetupControllerTest.java | 2 +- .../api/SetupStatusProjectionFactoryTest.java | 68 ++++++++-- ...dOptionalConfigurationPersistenceTest.java | 82 +++++++++++- .../UnattendedSetupInitializerTest.java | 6 +- .../workflow/DefaultSetupWorkflowTest.java | 31 ++++- .../HeadlessSetupCoordinatorTest.java | 6 +- .../workflow/SetupRequestValidatorTest.java | 57 +++++++-- .../workflow/SetupWarningPolicyTest.java | 16 ++- .../ManagedConfigDataPrecedenceTest.java | 79 ++++++++++++ 25 files changed, 627 insertions(+), 258 deletions(-) delete mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/PublicAccessConfigurationValidator.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ServerInstrumentationConfigurationValidator.java diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/impl/EmailAlertNotifyHandlerImpl.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/impl/EmailAlertNotifyHandlerImpl.java index f2d8f7eb02..60d44b9926 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/impl/EmailAlertNotifyHandlerImpl.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/impl/EmailAlertNotifyHandlerImpl.java @@ -54,6 +54,9 @@ public class EmailAlertNotifyHandlerImpl extends AbstractAlertNotifyHandlerImpl @Value("${spring.mail.username:demo}") private String username; + @Value("${hertzbeat.mail.from-address:${spring.mail.username:demo}}") + private String fromAddress; + @Value("${spring.mail.password:demo}") private String password; @@ -82,7 +85,7 @@ public class EmailAlertNotifyHandlerImpl extends AbstractAlertNotifyHandlerImpl try { // get sender JavaMailSenderImpl sender = (JavaMailSenderImpl) javaMailSender; - String fromUsername = username; + String fromUsername = fromAddress; try { boolean useDatabase = false; GeneralConfig emailConfig = generalConfigDao.findByType(TYPE); diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/impl/EmailAlertNotifyHandlerImplTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/impl/EmailAlertNotifyHandlerImplTest.java index 901b27fae5..47aea778ba 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/impl/EmailAlertNotifyHandlerImplTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/impl/EmailAlertNotifyHandlerImplTest.java @@ -17,6 +17,7 @@ package org.apache.hertzbeat.alert.notice.impl; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.lenient; @@ -46,6 +47,8 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.ResourceBundle; +import java.util.concurrent.atomic.AtomicReference; +import org.springframework.test.util.ReflectionTestUtils; /** * Test case for Email Alert Notify @@ -106,7 +109,7 @@ class EmailAlertNotifyHandlerImplTest { .content(JsonUtil.toJson(mailServerConfig)) .build(); when(generalConfigDao.findByType(any())).thenReturn(generalConfig); - when(mailSender.getJavaMailProperties()).thenReturn(new Properties()); + lenient().when(mailSender.getJavaMailProperties()).thenReturn(new Properties()); } @Test @@ -130,4 +133,30 @@ class EmailAlertNotifyHandlerImplTest { assertThrows(AlertNoticeException.class, () -> emailAlertNotifyHandler.send(receiver, template, groupAlert)); } + + @Test + void configuredFromAddressIsUsedByProductionHandler() throws Exception { + AtomicReference sent = new AtomicReference<>(); + JavaMailSenderImpl sender = new JavaMailSenderImpl() { + @Override + public void send(MimeMessage mimeMessage) { + sent.set(mimeMessage); + } + }; + when(generalConfigDao.findByType(any())).thenReturn(null); + EmailAlertNotifyHandlerImpl handler = new EmailAlertNotifyHandlerImpl(sender, generalConfigDao); + ReflectionTestUtils.setField(handler, "host", "smtp.example.test"); + ReflectionTestUtils.setField(handler, "port", 465); + ReflectionTestUtils.setField(handler, "username", "smtp-user@example.test"); + ReflectionTestUtils.setField(handler, "password", "password"); + ReflectionTestUtils.setField(handler, "fromAddress", "alerts@example.test"); + ReflectionTestUtils.setField(handler, "sslEnable", true); + ReflectionTestUtils.setField(handler, "starttlsEnable", false); + ReflectionTestUtils.setField(handler, "bundle", bundle); + when(bundle.getString("alerter.notify.title")).thenReturn("Alert Notification"); + + handler.send(receiver, template, groupAlert); + + assertEquals("alerts@example.test", sent.get().getFrom()[0].toString()); + } } diff --git a/hertzbeat-manager/pom.xml b/hertzbeat-manager/pom.xml index 549c402454..8d8a40349f 100644 --- a/hertzbeat-manager/pom.xml +++ b/hertzbeat-manager/pom.xml @@ -90,7 +90,6 @@ org.apache.hertzbeat hertzbeat-observability - test diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java index 2e72b4e7b7..3e1ccf96a8 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java @@ -193,7 +193,7 @@ public final class SetupApiContract { public enum ValidationSection implements WireValue { METADATA_DATABASE("metadata_database"), TELEMETRY_STORE("telemetry_store"), - PUBLIC_ACCESS("public_access"), + SERVER_INSTRUMENTATION("server_instrumentation"), MAIL("mail"); private final String value; @@ -244,7 +244,7 @@ public final class SetupApiContract { METADATA_SCHEMA_MISMATCH("metadata_schema_mismatch"), METADATA_INSUFFICIENT_PRIVILEGES("metadata_insufficient_privileges"), TELEMETRY_CONNECTION_FAILED("telemetry_connection_failed"), - PUBLIC_ADDRESS_INVALID("public_address_invalid"), + SERVER_INSTRUMENTATION_INVALID("server_instrumentation_invalid"), MAIL_CONNECTION_FAILED("mail_connection_failed"), ADMINISTRATOR_ALREADY_CONFIGURED("administrator_already_configured"), ADMINISTRATOR_USERNAME_INVALID("administrator_username_invalid"), @@ -274,7 +274,7 @@ public final class SetupApiContract { public enum SetupWarningCode implements WireValue { EXTERNAL_APPLY_REQUIRED("external_apply_required"), RESTART_REQUIRED("restart_required"), - PUBLIC_ADDRESS_PLAINTEXT("public_address_plaintext"), + SERVER_OTLP_PLAINTEXT("server_otlp_plaintext"), MAIL_SECURITY_NONE("mail_security_none"), H2_NON_PRODUCTION("h2_non_production"); @@ -338,7 +338,6 @@ public final class SetupApiContract { /** Secret-free optional configuration status. */ public record OptionalConfigurationSummary( - boolean publicAccessConfigured, boolean serverOtlpHttpConfigured, boolean serverOtlpGrpcConfigured, boolean retentionConfigured, @@ -408,9 +407,8 @@ public final class SetupApiContract { } } - /** Public endpoint input; HTTP and HTTPS are both contractually valid. */ - public record PublicAccessConfiguration( - String publicBaseUrl, + /** Server OTLP endpoint input; HTTP and HTTPS are both contractually valid. */ + public record ServerInstrumentationConfiguration( String serverOtlpHttpEndpoint, String serverOtlpGrpcEndpoint) { } @@ -442,16 +440,16 @@ public final class SetupApiContract { @NotNull ValidationSection section, @Valid MetadataDatabaseConfiguration managementDatabase, @Valid TelemetryStoreConfiguration telemetryStore, - @Valid PublicAccessConfiguration publicAccess, + @Valid ServerInstrumentationConfiguration serverInstrumentation, @Valid MailConfiguration mail) { public ValidateRequest { Objects.requireNonNull(section, "section"); - int supplied = countPresent(managementDatabase, telemetryStore, publicAccess, mail); + int supplied = countPresent(managementDatabase, telemetryStore, serverInstrumentation, mail); boolean matches = switch (section) { case METADATA_DATABASE -> managementDatabase != null; case TELEMETRY_STORE -> telemetryStore != null; - case PUBLIC_ACCESS -> publicAccess != null; + case SERVER_INSTRUMENTATION -> serverInstrumentation != null; case MAIL -> mail != null; }; if (supplied != 1 || !matches) { @@ -518,28 +516,24 @@ public final class SetupApiContract { } /** Optional retention input. */ - public record RetentionConfiguration( - @Positive Integer metricsDays, - @Positive Integer logsDays, - @Positive Integer tracesDays) { + public record RetentionConfiguration(@Positive int days) { public RetentionConfiguration { - requirePositiveIfPresent(metricsDays); - requirePositiveIfPresent(logsDays); - requirePositiveIfPresent(tracesDays); + if (days <= 0) { + throw new IllegalArgumentException("Retention must be positive"); + } } } /** Optional setup input. */ public record OptionsRequest( - @Valid PublicAccessConfiguration publicAccess, + @Valid ServerInstrumentationConfiguration serverInstrumentation, @Valid RetentionConfiguration retention, @Valid MailConfiguration mail) { } /** Secret-free optional setup result. */ public record OptionsResponse( - boolean publicAccessConfigured, boolean serverOtlpHttpConfigured, boolean serverOtlpGrpcConfigured, boolean retentionConfigured, @@ -609,12 +603,6 @@ public final class SetupApiContract { return count; } - private static void requirePositiveIfPresent(Integer value) { - if (value != null && value <= 0) { - throw new IllegalArgumentException("Retention days must be positive when supplied"); - } - } - private static boolean hasText(String value) { return value != null && !value.isBlank(); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupStatusProjectionFactory.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupStatusProjectionFactory.java index 0c1fb2e75b..26fbbcd2bf 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupStatusProjectionFactory.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupStatusProjectionFactory.java @@ -9,12 +9,10 @@ package org.apache.hertzbeat.manager.setup.api; import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.DATABASE_KIND; import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.GREPTIME_ENABLED; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.GREPTIME_EXPIRE_TIME; import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.MAIL_HOST; -import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.MAIL_SECURITY; -import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.PUBLIC_BASE_URL; -import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.RETENTION_LOGS; -import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.RETENTION_METRICS; -import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.RETENTION_TRACES; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.MAIL_SSL_ENABLED; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.MAIL_STARTTLS_ENABLED; import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.SERVER_OTLP_GRPC; import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.SERVER_OTLP_HTTP; @@ -29,6 +27,7 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreSum import org.apache.hertzbeat.manager.setup.config.EffectiveConfigurationResolver; import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector.Inspection; import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector.State; +import org.apache.hertzbeat.manager.setup.config.ManagedOptionalConfiguration.ServerInstrumentationSettings; import org.apache.hertzbeat.manager.setup.config.RestartRequirement; import org.apache.hertzbeat.manager.setup.workflow.SetupConfigurationProjection; import org.apache.hertzbeat.manager.setup.workflow.SetupWarningPolicy; @@ -48,30 +47,49 @@ final class SetupStatusProjectionFactory { : database.source(); boolean managedPresent = inspection.state() == State.LOADABLE; MetadataDatabaseKind kind = MetadataDatabaseKind.valueOf(database.value().toUpperCase(Locale.ROOT)); + boolean mailConfigured = externallyConfigured(environment, MAIL_HOST, true); OptionalConfigurationSummary optional = new OptionalConfigurationSummary( - hasText(environment, PUBLIC_BASE_URL), hasText(environment, SERVER_OTLP_HTTP), - hasText(environment, SERVER_OTLP_GRPC), hasRetention(environment), - hasText(environment, MAIL_HOST)); - String mailSecurityValue = environment.getProperty(MAIL_SECURITY); - MailSecurity mailSecurity = mailSecurityValue != null - && MailSecurity.NONE.name().equalsIgnoreCase(mailSecurityValue) ? MailSecurity.NONE : null; + externallyConfiguredEndpoint(environment, SERVER_OTLP_HTTP), + externallyConfiguredEndpoint(environment, SERVER_OTLP_GRPC), + externallyConfigured(environment, GREPTIME_EXPIRE_TIME, true), mailConfigured); + MailSecurity mailSecurity = mailConfigured ? mailSecurity(environment) : null; return new SetupConfigurationProjection( new ManagementDatabaseSummary(kind, managedPresent || database.source() != ConfigSource.BUILT_IN_DEFAULT, database.source(), false), new TelemetryStoreSummary(TelemetryStoreKind.GREPTIME, managedPresent || telemetrySource != ConfigSource.BUILT_IN_DEFAULT, telemetrySource, false), optional, SetupWarningPolicy.INSTANCE.evaluate( - kind, environment.getProperty(PUBLIC_BASE_URL), mailSecurity)); + kind, environment.getProperty(SERVER_OTLP_HTTP), + environment.getProperty(SERVER_OTLP_GRPC), mailSecurity)); } - private static boolean hasText(Environment environment, String key) { - String value = environment.getProperty(key); - return value != null && !value.isBlank(); + private boolean externallyConfigured(Environment environment, String key, boolean requireText) { + if (!environment.containsProperty(key)) { + return false; + } + var resolved = resolver.resolve(environment, key, RestartRequirement.LIVE_RELOAD); + return resolved.source() != ConfigSource.BUILT_IN_DEFAULT + && (!requireText || !resolved.value().isBlank()); } - private static boolean hasRetention(Environment environment) { - return environment.containsProperty(RETENTION_METRICS) - || environment.containsProperty(RETENTION_LOGS) - || environment.containsProperty(RETENTION_TRACES); + private boolean externallyConfiguredEndpoint(Environment environment, String key) { + if (!environment.containsProperty(key)) { + return false; + } + var resolved = resolver.resolve(environment, key, RestartRequirement.LIVE_RELOAD); + return resolved.source() != ConfigSource.BUILT_IN_DEFAULT + && ServerInstrumentationSettings.normalize(resolved.value()).isPresent(); + } + + private static MailSecurity mailSecurity(Environment environment) { + if (!environment.containsProperty(MAIL_SSL_ENABLED) + || !environment.containsProperty(MAIL_STARTTLS_ENABLED)) { + return null; + } + if (environment.getProperty(MAIL_SSL_ENABLED, Boolean.class, false)) { + return MailSecurity.TLS; + } + return environment.getProperty(MAIL_STARTTLS_ENABLED, Boolean.class, false) + ? MailSecurity.STARTTLS : MailSecurity.NONE; } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ApplicationConfigDocumentCodec.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ApplicationConfigDocumentCodec.java index 4d48b0ec77..e489a362d0 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ApplicationConfigDocumentCodec.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ApplicationConfigDocumentCodec.java @@ -22,17 +22,18 @@ import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.DATASOURCE_USERNAME; import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.GREPTIME_DATABASE; import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.GREPTIME_ENABLED; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.GREPTIME_EXPIRE_TIME; import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.GREPTIME_GRPC; import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.GREPTIME_HTTP; import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.GREPTIME_USERNAME; import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.MAIL_HOST; -import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.MAIL_SECURITY; -import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.PUBLIC_BASE_URL; -import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.RETENTION_LOGS; -import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.RETENTION_METRICS; -import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.RETENTION_TRACES; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.MAIL_FROM_ADDRESS; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.MAIL_SSL_ENABLED; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.MAIL_STARTTLS_ENABLED; import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.SERVER_OTLP_GRPC; import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.SERVER_OTLP_HTTP; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.SERVER_AUTHENTICATION; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.SERVER_PROFILE_ID; import java.util.LinkedHashMap; import java.util.Map; @@ -51,7 +52,8 @@ final class ApplicationConfigDocumentCodec implements ManagedDocumentCodec REQUIRED_KEYS = Set.of( DATASOURCE_URL, DATASOURCE_USERNAME, DATABASE_KIND, DUCKDB_ENABLED, GREPTIME_ENABLED, @@ -59,9 +61,9 @@ final class ApplicationConfigDocumentCodec implements ManagedDocumentCodec ALLOWED_KEYS = Set.of( DATASOURCE_URL, DATASOURCE_USERNAME, DATABASE_KIND, DUCKDB_ENABLED, GREPTIME_ENABLED, GREPTIME_GRPC, GREPTIME_HTTP, - GREPTIME_DATABASE, GREPTIME_USERNAME, PUBLIC_BASE_URL, SERVER_OTLP_HTTP, - SERVER_OTLP_GRPC, RETENTION_METRICS, RETENTION_LOGS, RETENTION_TRACES, - MAIL_HOST, MAIL_PORT, MAIL_SECURITY, MAIL_USERNAME, MAIL_FROM); + GREPTIME_DATABASE, GREPTIME_USERNAME, GREPTIME_EXPIRE_TIME, SERVER_OTLP_HTTP, + SERVER_OTLP_GRPC, SERVER_PROFILE_ID, SERVER_AUTHENTICATION, MAIL_HOST, MAIL_PORT, + MAIL_SSL_ENABLED, MAIL_STARTTLS_ENABLED, MAIL_USERNAME, MAIL_FROM_ADDRESS); @Override public byte[] encode(ManagedApplicationConfig value, String generation) { @@ -90,22 +92,21 @@ final class ApplicationConfigDocumentCodec implements ManagedDocumentCodec values.put(GREPTIME_USERNAME, username)); - value.optional().publicAccess().ifPresent(publicAccess -> { - publicAccess.publicBaseUrl().ifPresent(item -> values.put(PUBLIC_BASE_URL, item)); - publicAccess.serverOtlpHttpEndpoint().ifPresent(item -> values.put(SERVER_OTLP_HTTP, item)); - publicAccess.serverOtlpGrpcEndpoint().ifPresent(item -> values.put(SERVER_OTLP_GRPC, item)); - }); - value.optional().retention().ifPresent(retention -> { - putInteger(values, RETENTION_METRICS, retention.metricsDays()); - putInteger(values, RETENTION_LOGS, retention.logsDays()); - putInteger(values, RETENTION_TRACES, retention.tracesDays()); + value.optional().serverInstrumentation().ifPresent(instrumentation -> { + values.put(SERVER_PROFILE_ID, MANAGED_SERVER_PROFILE_ID); + values.put(SERVER_AUTHENTICATION, MANAGED_SERVER_AUTHENTICATION); + instrumentation.serverOtlpHttpEndpoint().ifPresent(item -> values.put(SERVER_OTLP_HTTP, item)); + instrumentation.serverOtlpGrpcEndpoint().ifPresent(item -> values.put(SERVER_OTLP_GRPC, item)); }); + value.optional().retention().ifPresent(retention -> + values.put(GREPTIME_EXPIRE_TIME, retention.days() + "d")); value.optional().mail().ifPresent(mail -> { values.put(MAIL_HOST, mail.host()); values.put(MAIL_PORT, Integer.toString(mail.port())); - values.put(MAIL_SECURITY, mail.security().name()); + values.put(MAIL_SSL_ENABLED, Boolean.toString(mail.security() == MailSecurity.TLS)); + values.put(MAIL_STARTTLS_ENABLED, Boolean.toString(mail.security() == MailSecurity.STARTTLS)); mail.username().ifPresent(item -> values.put(MAIL_USERNAME, item)); - values.put(MAIL_FROM, mail.fromAddress()); + values.put(MAIL_FROM_ADDRESS, mail.fromAddress()); }); return values; } @@ -128,6 +129,7 @@ final class ApplicationConfigDocumentCodec implements ManagedDocumentCodec values) { - boolean publicPresent = containsAny(values, PUBLIC_BASE_URL, SERVER_OTLP_HTTP, SERVER_OTLP_GRPC); - Optional publicAccess = publicPresent - ? Optional.of(new ManagedOptionalConfiguration.PublicAccessSettings( - optionalText(values, PUBLIC_BASE_URL), optionalText(values, SERVER_OTLP_HTTP), - optionalText(values, SERVER_OTLP_GRPC))) : Optional.empty(); - boolean retentionPresent = containsAny(values, RETENTION_METRICS, RETENTION_LOGS, RETENTION_TRACES); - Optional retention = retentionPresent - ? Optional.of(new ManagedOptionalConfiguration.RetentionSettings( - optionalInteger(values, RETENTION_METRICS), optionalInteger(values, RETENTION_LOGS), - optionalInteger(values, RETENTION_TRACES))) : Optional.empty(); + boolean instrumentationPresent = containsAny(values, SERVER_OTLP_HTTP, SERVER_OTLP_GRPC); + Optional instrumentation = + instrumentationPresent ? Optional.of(new ManagedOptionalConfiguration.ServerInstrumentationSettings( + optionalText(values, SERVER_OTLP_HTTP), optionalText(values, SERVER_OTLP_GRPC))) + : Optional.empty(); + Optional retention = values.containsKey(GREPTIME_EXPIRE_TIME) + ? Optional.of(new ManagedOptionalConfiguration.RetentionSettings(retentionDays(values))) + : Optional.empty(); Optional mail = values.containsKey(MAIL_HOST) ? Optional.of(new ManagedOptionalConfiguration.MailSettings( - text(values, MAIL_HOST), Integer.parseInt(text(values, MAIL_PORT)), - MailSecurity.valueOf(text(values, MAIL_SECURITY)), optionalText(values, MAIL_USERNAME), - text(values, MAIL_FROM))) : Optional.empty(); - return new ManagedOptionalConfiguration(publicAccess, retention, mail); + text(values, MAIL_HOST), Integer.parseInt(text(values, MAIL_PORT)), + mailSecurity(values), optionalText(values, MAIL_USERNAME), + text(values, MAIL_FROM_ADDRESS))) : Optional.empty(); + return new ManagedOptionalConfiguration(instrumentation, retention, mail); } private static boolean completeMailGroup(Map values) { - boolean any = containsAny(values, MAIL_HOST, MAIL_PORT, MAIL_SECURITY, MAIL_USERNAME, MAIL_FROM); - return !any || values.keySet().containsAll(Set.of(MAIL_HOST, MAIL_PORT, MAIL_SECURITY, MAIL_FROM)); + boolean any = containsAny(values, MAIL_HOST, MAIL_PORT, MAIL_SSL_ENABLED, MAIL_STARTTLS_ENABLED, + MAIL_USERNAME, MAIL_FROM_ADDRESS); + return !any || values.keySet().containsAll(Set.of(MAIL_HOST, MAIL_PORT, MAIL_SSL_ENABLED, + MAIL_STARTTLS_ENABLED, MAIL_FROM_ADDRESS)); + } + + private static boolean completeServerInstrumentationGroup(Map values) { + boolean endpointKeyPresent = containsAny(values, SERVER_OTLP_HTTP, SERVER_OTLP_GRPC); + boolean endpointPresent = meaningfulEndpoint(values, SERVER_OTLP_HTTP) + || meaningfulEndpoint(values, SERVER_OTLP_GRPC); + boolean endpointValuesValid = (!values.containsKey(SERVER_OTLP_HTTP) + || meaningfulEndpoint(values, SERVER_OTLP_HTTP)) + && (!values.containsKey(SERVER_OTLP_GRPC) + || meaningfulEndpoint(values, SERVER_OTLP_GRPC)); + boolean internalPresent = containsAny(values, SERVER_PROFILE_ID, SERVER_AUTHENTICATION); + return (!endpointKeyPresent && !internalPresent) || (endpointPresent && endpointValuesValid + && values.keySet().containsAll(Set.of(SERVER_PROFILE_ID, SERVER_AUTHENTICATION)) + && MANAGED_SERVER_PROFILE_ID.equals(values.get(SERVER_PROFILE_ID)) + && MANAGED_SERVER_AUTHENTICATION.equals(values.get(SERVER_AUTHENTICATION))); + } + + private static boolean meaningfulEndpoint(Map values, String key) { + return values.get(key) instanceof String endpoint + && ManagedOptionalConfiguration.ServerInstrumentationSettings.normalize(endpoint).isPresent(); } private static boolean containsAny(Map values, String... keys) { @@ -187,14 +209,29 @@ final class ApplicationConfigDocumentCodec implements ManagedDocumentCodec values, String key) { - return values.containsKey(key) ? Integer.valueOf(text(values, key)) : null; + private static int retentionDays(Map values) { + String value = text(values, GREPTIME_EXPIRE_TIME); + if (!value.endsWith("d")) { + throw new IllegalArgumentException("Managed retention is invalid"); + } + return Integer.parseInt(value.substring(0, value.length() - 1)); } - private static void putInteger(Map values, String key, Integer value) { - if (value != null) { - values.put(key, value.toString()); + private static MailSecurity mailSecurity(Map values) { + boolean ssl = booleanValue(values, MAIL_SSL_ENABLED); + boolean startTls = booleanValue(values, MAIL_STARTTLS_ENABLED); + if (ssl && startTls) { + throw new IllegalArgumentException("Mail security settings conflict"); } + return ssl ? MailSecurity.TLS : startTls ? MailSecurity.STARTTLS : MailSecurity.NONE; + } + + private static boolean booleanValue(Map values, String key) { + String value = text(values, key); + if (!"true".equals(value) && !"false".equals(value)) { + throw new IllegalArgumentException("Managed boolean is invalid"); + } + return Boolean.parseBoolean(value); } /** diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationKeys.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationKeys.java index bb43ac2403..7ec81ac4a9 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationKeys.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationKeys.java @@ -29,14 +29,15 @@ public final class ManagedConfigurationKeys { public static final String GREPTIME_DATABASE = "warehouse.store.greptime.database"; public static final String GREPTIME_USERNAME = "warehouse.store.greptime.username"; public static final String GREPTIME_PASSWORD = "warehouse.store.greptime.password"; - public static final String PUBLIC_BASE_URL = "hertzbeat.setup.public-base-url"; - public static final String SERVER_OTLP_HTTP = "hertzbeat.setup.server-otlp-http-endpoint"; - public static final String SERVER_OTLP_GRPC = "hertzbeat.setup.server-otlp-grpc-endpoint"; - public static final String RETENTION_METRICS = "hertzbeat.setup.retention.metrics-days"; - public static final String RETENTION_LOGS = "hertzbeat.setup.retention.logs-days"; - public static final String RETENTION_TRACES = "hertzbeat.setup.retention.traces-days"; + public static final String SERVER_OTLP_HTTP = "hertzbeat.instrumentation.server.otlp-http-endpoint"; + public static final String SERVER_OTLP_GRPC = "hertzbeat.instrumentation.server.otlp-grpc-endpoint"; + public static final String SERVER_PROFILE_ID = "hertzbeat.instrumentation.server.profile-id"; + public static final String SERVER_AUTHENTICATION = "hertzbeat.instrumentation.server.authentication"; + public static final String GREPTIME_EXPIRE_TIME = "warehouse.store.greptime.expire-time"; public static final String MAIL_HOST = "spring.mail.host"; - public static final String MAIL_SECURITY = "hertzbeat.setup.mail.security"; + public static final String MAIL_SSL_ENABLED = "spring.mail.properties.mail.smtp.ssl.enable"; + public static final String MAIL_STARTTLS_ENABLED = "spring.mail.properties.mail.smtp.starttls.enable"; + public static final String MAIL_FROM_ADDRESS = "hertzbeat.mail.from-address"; private ManagedConfigurationKeys() { } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionalConfiguration.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionalConfiguration.java index 9beaf0ddcb..00631ec179 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionalConfiguration.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionalConfiguration.java @@ -23,12 +23,12 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; /** Typed optional setup overlay kept in the existing managed application document. */ public record ManagedOptionalConfiguration( - Optional publicAccess, + Optional serverInstrumentation, Optional retention, Optional mail) { public ManagedOptionalConfiguration { - Objects.requireNonNull(publicAccess, "publicAccess"); + Objects.requireNonNull(serverInstrumentation, "serverInstrumentation"); Objects.requireNonNull(retention, "retention"); Objects.requireNonNull(mail, "mail"); } @@ -37,28 +37,40 @@ public record ManagedOptionalConfiguration( return new ManagedOptionalConfiguration(Optional.empty(), Optional.empty(), Optional.empty()); } - /** Optional externally advertised operator and OTLP endpoints. */ - public record PublicAccessSettings( - Optional publicBaseUrl, + /** Optional server OTLP intake endpoints. */ + public record ServerInstrumentationSettings( Optional serverOtlpHttpEndpoint, Optional serverOtlpGrpcEndpoint) { - public PublicAccessSettings { - Objects.requireNonNull(publicBaseUrl, "publicBaseUrl"); + public ServerInstrumentationSettings { Objects.requireNonNull(serverOtlpHttpEndpoint, "serverOtlpHttpEndpoint"); Objects.requireNonNull(serverOtlpGrpcEndpoint, "serverOtlpGrpcEndpoint"); + serverOtlpHttpEndpoint = normalizeConfigured(serverOtlpHttpEndpoint); + serverOtlpGrpcEndpoint = normalizeConfigured(serverOtlpGrpcEndpoint); + if (serverOtlpHttpEndpoint.isEmpty() && serverOtlpGrpcEndpoint.isEmpty()) { + throw new IllegalArgumentException("At least one server instrumentation endpoint is required"); + } + } + + public static Optional normalize(String value) { + if (value == null) { + return Optional.empty(); + } + String normalized = value.trim(); + return normalized.isEmpty() ? Optional.empty() : Optional.of(normalized); + } + + private static Optional normalizeConfigured(Optional endpoint) { + if (endpoint.isPresent() && normalize(endpoint.orElseThrow()).isEmpty()) { + throw new IllegalArgumentException("Server instrumentation endpoint must not be blank"); + } + return endpoint.flatMap(ServerInstrumentationSettings::normalize); } } - /** Optional retention periods by signal family. */ - public record RetentionSettings(Integer metricsDays, Integer logsDays, Integer tracesDays) { + /** Optional Greptime database retention period. */ + public record RetentionSettings(int days) { public RetentionSettings { - requirePositive(metricsDays); - requirePositive(logsDays); - requirePositive(tracesDays); - } - - private static void requirePositive(Integer days) { - if (days != null && days <= 0) { + if (days <= 0) { throw new IllegalArgumentException("Retention must be positive"); } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflow.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflow.java index 19b8d7a616..0f9a237f79 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflow.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflow.java @@ -40,6 +40,7 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationRespons import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationSection; import org.apache.hertzbeat.manager.setup.api.SetupApiException; import org.apache.hertzbeat.manager.setup.api.SetupWorkflow; +import org.apache.hertzbeat.manager.setup.config.ManagedOptionalConfiguration.ServerInstrumentationSettings; import org.springframework.http.HttpStatus; /** Cohesive setup state-machine facade; transport and persistence remain in dedicated collaborators. */ @@ -115,9 +116,9 @@ public final class DefaultSetupWorkflow implements SetupWorkflow { private OptionsResponse configureOptionsMutation(OptionsRequest request) { requireWritable(); state.ensurePhase(SetupPhase.OPTIONAL_CONFIGURATION); - if (request.publicAccess() != null) { - requireValid(new ValidateRequest(ValidationSection.PUBLIC_ACCESS, - null, null, request.publicAccess(), null)); + if (request.serverInstrumentation() != null) { + requireValid(new ValidateRequest(ValidationSection.SERVER_INSTRUMENTATION, + null, null, request.serverInstrumentation(), null)); } if (request.mail() != null) { requireValid(new ValidateRequest(ValidationSection.MAIL, @@ -125,14 +126,17 @@ public final class DefaultSetupWorkflow implements SetupWorkflow { } options.persist(request); OptionalConfigurationSummary summary = new OptionalConfigurationSummary( - request.publicAccess() != null && hasText(request.publicAccess().publicBaseUrl()), - request.publicAccess() != null && hasText(request.publicAccess().serverOtlpHttpEndpoint()), - request.publicAccess() != null && hasText(request.publicAccess().serverOtlpGrpcEndpoint()), + request.serverInstrumentation() != null + && ServerInstrumentationSettings.normalize( + request.serverInstrumentation().serverOtlpHttpEndpoint()).isPresent(), + request.serverInstrumentation() != null + && ServerInstrumentationSettings.normalize( + request.serverInstrumentation().serverOtlpGrpcEndpoint()).isPresent(), request.retention() != null, request.mail() != null); state.optionsConfigured(summary, SetupWarningPolicy.INSTANCE.evaluate(state.managementDatabaseKind(), request)); - return new OptionsResponse(summary.publicAccessConfigured(), summary.serverOtlpHttpConfigured(), - summary.serverOtlpGrpcConfigured(), summary.retentionConfigured(), summary.mailConfigured(), + return new OptionsResponse(summary.serverOtlpHttpConfigured(), summary.serverOtlpGrpcConfigured(), + summary.retentionConfigured(), summary.mailConfigured(), SetupPhase.OPTIONAL_CONFIGURATION); } @@ -169,7 +173,4 @@ public final class DefaultSetupWorkflow implements SetupWorkflow { } } - private static boolean hasText(String value) { - return value != null && !value.isBlank(); - } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/PublicAccessConfigurationValidator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/PublicAccessConfigurationValidator.java deleted file mode 100644 index abf91a0714..0000000000 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/PublicAccessConfigurationValidator.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hertzbeat.manager.setup.workflow; - -import java.net.URI; -import java.util.ArrayList; -import java.util.List; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.PublicAccessConfiguration; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; -import org.apache.hertzbeat.manager.setup.workflow.MetadataConfigurationValidator.Validation; - -/** Validates optional public endpoint URIs and reports plaintext exposure explicitly. */ -final class PublicAccessConfigurationValidator { - Validation validate(PublicAccessConfiguration configuration) { - List warnings = new ArrayList<>(); - for (String value : List.of(nullToEmpty(configuration.publicBaseUrl()), - nullToEmpty(configuration.serverOtlpHttpEndpoint()))) { - if (value.isEmpty()) { - continue; - } - URI uri; - try { - uri = URI.create(value); - } catch (IllegalArgumentException failure) { - return Validation.failed(SetupErrorCode.PUBLIC_ADDRESS_INVALID); - } - if (uri.getHost() == null || !("http".equalsIgnoreCase(uri.getScheme()) - || "https".equalsIgnoreCase(uri.getScheme()))) { - return Validation.failed(SetupErrorCode.PUBLIC_ADDRESS_INVALID); - } - if ("http".equalsIgnoreCase(uri.getScheme())) { - warnings.add(SetupWarningCode.PUBLIC_ADDRESS_PLAINTEXT); - } - } - String grpc = configuration.serverOtlpGrpcEndpoint(); - if (grpc != null && !grpc.isBlank() && !validHostPort(grpc)) { - return Validation.failed(SetupErrorCode.PUBLIC_ADDRESS_INVALID); - } - return new Validation(true, null, List.copyOf(warnings)); - } - - private static boolean validHostPort(String value) { - int separator = value.lastIndexOf(':'); - if (separator < 1 || separator == value.length() - 1 || value.indexOf('/') >= 0 - || value.substring(0, separator).isBlank() || value.substring(0, separator).contains(" ")) { - return false; - } - try { - int port = Integer.parseInt(value.substring(separator + 1)); - return port > 0 && port <= 65_535; - } catch (NumberFormatException failure) { - return false; - } - } - - private static String nullToEmpty(String value) { - return value == null ? "" : value; - } -} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ServerInstrumentationConfigurationValidator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ServerInstrumentationConfigurationValidator.java new file mode 100644 index 0000000000..f42eb90ff3 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ServerInstrumentationConfigurationValidator.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.List; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ServerInstrumentationConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; +import org.apache.hertzbeat.manager.setup.config.ManagedOptionalConfiguration.ServerInstrumentationSettings; +import org.apache.hertzbeat.manager.setup.workflow.MetadataConfigurationValidator.Validation; +import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.IntakeEndpoint; + +/** Validates optional server OTLP intake endpoints. */ +final class ServerInstrumentationConfigurationValidator { + Validation validate(ServerInstrumentationConfiguration configuration) { + String http = ServerInstrumentationSettings.normalize( + configuration.serverOtlpHttpEndpoint()).orElse(null); + String grpc = ServerInstrumentationSettings.normalize( + configuration.serverOtlpGrpcEndpoint()).orElse(null); + if ((http == null && grpc == null) || !validEndpoint(http) || !validEndpoint(grpc)) { + return Validation.failed(SetupErrorCode.SERVER_INSTRUMENTATION_INVALID); + } + List warnings = plaintext(http) || plaintext(grpc) + ? List.of(SetupWarningCode.SERVER_OTLP_PLAINTEXT) : List.of(); + return new Validation(true, null, warnings); + } + + private static boolean validEndpoint(String value) { + if (value == null || value.isBlank()) { + return true; + } + try { + IntakeEndpoint.fromUrl(value); + return true; + } catch (IllegalArgumentException failure) { + return false; + } + } + + private static boolean plaintext(String value) { + return value != null && value.regionMatches(true, 0, "http://", 0, 7); + } + +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationProjection.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationProjection.java index bda9badc55..6e9d7d7858 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationProjection.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationProjection.java @@ -33,7 +33,7 @@ public record SetupConfigurationProjection( ConfigSource.BUILT_IN_DEFAULT, false), new TelemetryStoreSummary(TelemetryStoreKind.GREPTIME, false, ConfigSource.BUILT_IN_DEFAULT, false), - new OptionalConfigurationSummary(false, false, false, false, false), + new OptionalConfigurationSummary(false, false, false, false), List.of(SetupWarningCode.H2_NON_PRODUCTION)); } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOptionsCoordinator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOptionsCoordinator.java index caabf0221d..1dce88e28a 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOptionsCoordinator.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOptionsCoordinator.java @@ -37,13 +37,17 @@ public final class SetupOptionsCoordinator { public void persist(OptionsRequest request) { ManagedOptionalConfiguration options = new ManagedOptionalConfiguration( - Optional.ofNullable(request.publicAccess()).map(value -> - new ManagedOptionalConfiguration.PublicAccessSettings( - text(value.publicBaseUrl()), text(value.serverOtlpHttpEndpoint()), - text(value.serverOtlpGrpcEndpoint()))), + Optional.ofNullable(request.serverInstrumentation()).flatMap(value -> { + Optional httpEndpoint = ManagedOptionalConfiguration.ServerInstrumentationSettings + .normalize(value.serverOtlpHttpEndpoint()); + Optional grpcEndpoint = ManagedOptionalConfiguration.ServerInstrumentationSettings + .normalize(value.serverOtlpGrpcEndpoint()); + return httpEndpoint.isEmpty() && grpcEndpoint.isEmpty() ? Optional.empty() + : Optional.of(new ManagedOptionalConfiguration.ServerInstrumentationSettings( + httpEndpoint, grpcEndpoint)); + }), Optional.ofNullable(request.retention()).map(value -> - new ManagedOptionalConfiguration.RetentionSettings( - value.metricsDays(), value.logsDays(), value.tracesDays())), + new ManagedOptionalConfiguration.RetentionSettings(value.days())), Optional.ofNullable(request.mail()).map(value -> new ManagedOptionalConfiguration.MailSettings(value.host(), value.port(), value.security(), text(value.username()), value.fromAddress()))); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidator.java index c129bda248..dead900e91 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidator.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidator.java @@ -32,7 +32,8 @@ public final class SetupRequestValidator { private final Clock clock; private final MetadataConfigurationValidator metadata = new MetadataConfigurationValidator(); private final TelemetryConfigurationValidator telemetry = new TelemetryConfigurationValidator(); - private final PublicAccessConfigurationValidator publicAccess = new PublicAccessConfigurationValidator(); + private final ServerInstrumentationConfigurationValidator serverInstrumentation = + new ServerInstrumentationConfigurationValidator(); private final MailConfigurationValidator mail = new MailConfigurationValidator(); private final MetadataConnectionProbe metadataConnection; private final TelemetryConnectionProbe telemetryConnection; @@ -57,7 +58,7 @@ public final class SetupRequestValidator { Validation structural = switch (request.section()) { case METADATA_DATABASE -> metadata.validate(request.managementDatabase()); case TELEMETRY_STORE -> telemetry.validate(request.telemetryStore()); - case PUBLIC_ACCESS -> publicAccess.validate(request.publicAccess()); + case SERVER_INSTRUMENTATION -> serverInstrumentation.validate(request.serverInstrumentation()); case MAIL -> mail.validate(request.mail()); }; Validation result = structural.valid() ? liveValidation(request, structural) : structural; @@ -86,7 +87,7 @@ public final class SetupRequestValidator { case METADATA_DATABASE -> metadataConnection.probe(request.managementDatabase()); case TELEMETRY_STORE -> telemetryConnection.probe(request.telemetryStore()); case MAIL -> mailConnection.probe(request.mail()); - case PUBLIC_ACCESS -> Optional.empty(); + case SERVER_INSTRUMENTATION -> Optional.empty(); }; return failure.map(Validation::failed).orElse(structural); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupWarningPolicy.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupWarningPolicy.java index 9ba81424b0..6ba9780837 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupWarningPolicy.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupWarningPolicy.java @@ -19,11 +19,11 @@ package org.apache.hertzbeat.manager.setup.workflow; import java.util.ArrayList; import java.util.List; -import java.util.Locale; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsRequest; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; +import org.apache.hertzbeat.manager.setup.config.ManagedOptionalConfiguration.ServerInstrumentationSettings; /** Single warning policy shared by live setup and restart status projection. */ public final class SetupWarningPolicy { @@ -33,23 +33,33 @@ public final class SetupWarningPolicy { } public List evaluate(MetadataDatabaseKind kind, OptionsRequest options) { - String publicUrl = options.publicAccess() == null ? null : options.publicAccess().publicBaseUrl(); + String otlpHttpEndpoint = options.serverInstrumentation() == null ? null + : options.serverInstrumentation().serverOtlpHttpEndpoint(); + String otlpGrpcEndpoint = options.serverInstrumentation() == null ? null + : options.serverInstrumentation().serverOtlpGrpcEndpoint(); MailSecurity mailSecurity = options.mail() == null ? null : options.mail().security(); - return evaluate(kind, publicUrl, mailSecurity); + return evaluate(kind, otlpHttpEndpoint, otlpGrpcEndpoint, mailSecurity); } public List evaluate( - MetadataDatabaseKind kind, String publicUrl, MailSecurity mailSecurity) { + MetadataDatabaseKind kind, String otlpHttpEndpoint, String otlpGrpcEndpoint, + MailSecurity mailSecurity) { List warnings = new ArrayList<>(); if (kind == MetadataDatabaseKind.H2) { warnings.add(SetupWarningCode.H2_NON_PRODUCTION); } - if (publicUrl != null && publicUrl.toLowerCase(Locale.ROOT).startsWith("http://")) { - warnings.add(SetupWarningCode.PUBLIC_ADDRESS_PLAINTEXT); + if (plaintext(otlpHttpEndpoint) || plaintext(otlpGrpcEndpoint)) { + warnings.add(SetupWarningCode.SERVER_OTLP_PLAINTEXT); } if (mailSecurity == MailSecurity.NONE) { warnings.add(SetupWarningCode.MAIL_SECURITY_NONE); } return List.copyOf(warnings); } + + private static boolean plaintext(String endpoint) { + return ServerInstrumentationSettings.normalize(endpoint) + .filter(value -> value.regionMatches(true, 0, "http://", 0, 7)) + .isPresent(); + } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java index d28729da9e..c08ab602d8 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java @@ -34,7 +34,7 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailConfiguration import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.PublicAccessConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ServerInstrumentationConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupAccess; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; @@ -75,11 +75,12 @@ class SetupApiContractTest { "awaiting_restart", "succeeded", "failed", "rolled_back"); assertWireValues(MetadataDatabaseKind.values(), "h2", "mysql", "postgresql"); assertWireValues(TelemetryStoreKind.values(), "greptime"); - assertWireValues(ValidationSection.values(), "metadata_database", "telemetry_store", "public_access", "mail"); + assertWireValues(ValidationSection.values(), "metadata_database", "telemetry_store", + "server_instrumentation", "mail"); assertWireValues(MailSecurity.values(), "none", "starttls", "tls"); assertWireValues(SetupApiContract.ExportFormat.values(), "yaml", "env", "kubernetes_secret"); assertWireValues(SetupApiContract.SetupWarningCode.values(), "external_apply_required", "restart_required", - "public_address_plaintext", "mail_security_none", "h2_non_production"); + "server_otlp_plaintext", "mail_security_none", "h2_non_production"); } @Test @@ -91,12 +92,12 @@ class SetupApiContractTest { "restartRequired"); assertComponents(SetupApiContract.TelemetryStoreSummary.class, "kind", "configured", "source", "restartRequired"); - assertComponents(SetupApiContract.OptionalConfigurationSummary.class, "publicAccessConfigured", - "serverOtlpHttpConfigured", "serverOtlpGrpcConfigured", "retentionConfigured", "mailConfigured"); + assertComponents(SetupApiContract.OptionalConfigurationSummary.class, "serverOtlpHttpConfigured", + "serverOtlpGrpcConfigured", "retentionConfigured", "mailConfigured"); assertComponents(SetupApiContract.UnlockRequest.class, "code"); assertComponents(SetupApiContract.UnlockResponse.class, "access", "expiresAt"); assertComponents(SetupApiContract.ValidateRequest.class, "section", "managementDatabase", "telemetryStore", - "publicAccess", "mail"); + "serverInstrumentation", "mail"); assertComponents(SetupApiContract.TelemetryStoreConfiguration.class, "kind", "grpcEndpoints", "httpEndpoint", "database", "username", "password"); assertComponents(SetupApiContract.ValidationResponse.class, "valid", "observedAt", "errorCode", "warnings"); @@ -108,9 +109,9 @@ class SetupApiContractTest { "startedAt", "completedAt", "errorCode", "nextPollAfterMillis", "exportAvailable"); assertComponents(SetupApiContract.AdministratorRequest.class, "username", "password"); assertComponents(SetupApiContract.AdministratorResponse.class, "username", "phase"); - assertComponents(SetupApiContract.OptionsRequest.class, "publicAccess", "retention", "mail"); - assertComponents(SetupApiContract.RetentionConfiguration.class, "metricsDays", "logsDays", "tracesDays"); - assertComponents(SetupApiContract.OptionsResponse.class, "publicAccessConfigured", "serverOtlpHttpConfigured", + assertComponents(SetupApiContract.OptionsRequest.class, "serverInstrumentation", "retention", "mail"); + assertComponents(SetupApiContract.RetentionConfiguration.class, "days"); + assertComponents(SetupApiContract.OptionsResponse.class, "serverOtlpHttpConfigured", "serverOtlpGrpcConfigured", "retentionConfigured", "mailConfigured", "phase"); assertComponents(SetupApiContract.ExportRequest.class, "format", "configuration"); assertComponents(SetupApiContract.ExportResponse.class, "fileName", "mediaType"); @@ -156,7 +157,7 @@ class SetupApiContractTest { () -> new ValidateRequest(ValidationSection.METADATA_DATABASE, null, null, null, null)); assertThrows(IllegalArgumentException.class, () -> new ValidateRequest( ValidationSection.METADATA_DATABASE, metadata, null, - new PublicAccessConfiguration("http://localhost:1157", null, null), null)); + new ServerInstrumentationConfiguration("http://localhost:4318", null), null)); assertThrows(IllegalArgumentException.class, () -> new ValidateRequest( ValidationSection.MAIL, metadata, null, null, null)); } @@ -184,7 +185,7 @@ class SetupApiContractTest { "config_read_only", "config_write_failed", "config_recovery_required", "metadata_connection_failed", "metadata_kind_unsupported", "metadata_schema_mismatch", "metadata_insufficient_privileges", "telemetry_connection_failed", - "public_address_invalid", "mail_connection_failed", "administrator_already_configured", + "server_instrumentation_invalid", "mail_connection_failed", "administrator_already_configured", "administrator_username_invalid", "operation_not_found", "operation_conflict", "migration_source_unsupported", "migration_target_not_empty", "migration_multi_node_unsupported", "migration_copy_failed", "migration_verification_failed", "migration_activation_failed", @@ -206,7 +207,7 @@ class SetupApiContractTest { new SetupApiContract.TelemetryStoreSummary( TelemetryStoreKind.GREPTIME, false, ConfigSource.BUILT_IN_DEFAULT, false), false, - new SetupApiContract.OptionalConfigurationSummary(false, false, false, false, false)); + new SetupApiContract.OptionalConfigurationSummary(false, false, false, false)); String json = objectMapper.writeValueAsString(response); assertFalse(json.contains("jdbc")); assertFalse(json.contains("username")); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupControllerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupControllerTest.java index 69eee3b206..fa533600ff 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupControllerTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupControllerTest.java @@ -76,7 +76,7 @@ class SetupControllerTest { ConfigSource.BUILT_IN_DEFAULT, false), new TelemetryStoreSummary(TelemetryStoreKind.GREPTIME, false, ConfigSource.BUILT_IN_DEFAULT, false), - false, new OptionalConfigurationSummary(false, false, false, false, false))); + false, new OptionalConfigurationSummary(false, false, false, false))); mvc.perform(get(SetupApiContract.STATUS_PATH)) .andExpect(status().isOk()) diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupStatusProjectionFactoryTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupStatusProjectionFactoryTest.java index 4133e415a5..33b74edb11 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupStatusProjectionFactoryTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupStatusProjectionFactoryTest.java @@ -10,6 +10,9 @@ package org.apache.hertzbeat.manager.setup.api; import static org.assertj.core.api.Assertions.assertThat; import java.util.Map; +import org.springframework.boot.env.OriginTrackedMapPropertySource; +import org.springframework.boot.origin.OriginTrackedValue; +import org.springframework.boot.origin.TextResourceOrigin; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigSource; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; @@ -17,8 +20,58 @@ import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspe import org.junit.jupiter.api.Test; import org.springframework.core.env.MapPropertySource; import org.springframework.core.env.StandardEnvironment; +import org.springframework.core.io.ClassPathResource; class SetupStatusProjectionFactoryTest { + @Test + void builtInRetentionAndMailDefaultsAreNotReportedAsConfigured() { + StandardEnvironment environment = new StandardEnvironment(); + TextResourceOrigin origin = new TextResourceOrigin(new ClassPathResource("application.yml"), + new TextResourceOrigin.Location(1, 1)); + environment.getPropertySources().addLast(new OriginTrackedMapPropertySource("built-in", Map.of( + "spring.jpa.database", OriginTrackedValue.of("H2", origin), + "warehouse.store.greptime.enabled", OriginTrackedValue.of("true", origin), + "warehouse.store.greptime.expire-time", OriginTrackedValue.of("30d", origin), + "spring.mail.host", OriginTrackedValue.of("smtp.qq.com", origin)))); + var inspection = new ManagedActiveConfigurationInspector.Inspection( + ManagedActiveConfigurationInspector.State.ABSENT, Map.of(), Map.of()); + + var projection = new SetupStatusProjectionFactory().create(environment, inspection); + + assertThat(projection.optional().retentionConfigured()).isFalse(); + assertThat(projection.optional().mailConfigured()).isFalse(); + } + + @Test + void blankManagedRetentionIsNotReportedAsConfigured() { + StandardEnvironment environment = new StandardEnvironment(); + environment.getPropertySources().addLast(new MapPropertySource( + ManagedActiveConfigurationInspector.MANAGED_APPLICATION_SOURCE, + Map.of("spring.jpa.database", "H2", "warehouse.store.greptime.enabled", "true", + "warehouse.store.greptime.expire-time", " "))); + var inspection = new ManagedActiveConfigurationInspector.Inspection( + ManagedActiveConfigurationInspector.State.LOADABLE, Map.of(), Map.of()); + + var projection = new SetupStatusProjectionFactory().create(environment, inspection); + + assertThat(projection.optional().retentionConfigured()).isFalse(); + } + + @Test + void controlOnlyManagedServerEndpointIsNotReportedAsConfigured() { + StandardEnvironment environment = new StandardEnvironment(); + environment.getPropertySources().addLast(new MapPropertySource( + ManagedActiveConfigurationInspector.MANAGED_APPLICATION_SOURCE, + Map.of("spring.jpa.database", "H2", "warehouse.store.greptime.enabled", "true", + "hertzbeat.instrumentation.server.otlp-http-endpoint", "\u0000"))); + var inspection = new ManagedActiveConfigurationInspector.Inspection( + ManagedActiveConfigurationInspector.State.LOADABLE, Map.of(), Map.of()); + + var projection = new SetupStatusProjectionFactory().create(environment, inspection); + + assertThat(projection.optional().serverOtlpHttpConfigured()).isFalse(); + } + @Test void restartProjectionUsesEffectiveSourceAndRehydratesSafeManagedOptions() { StandardEnvironment environment = new StandardEnvironment(); @@ -30,10 +83,11 @@ class SetupStatusProjectionFactoryTest { ManagedActiveConfigurationInspector.MANAGED_APPLICATION_SOURCE, Map.of("spring.jpa.database", "H2", "warehouse.store.greptime.enabled", "true", - "hertzbeat.setup.public-base-url", "http://localhost:1157", - "hertzbeat.setup.retention.metrics-days", "30", + "hertzbeat.instrumentation.server.otlp-http-endpoint", "http://localhost:4318", + "warehouse.store.greptime.expire-time", "30d", "spring.mail.host", "mail.example.test", - "hertzbeat.setup.mail.security", "NONE"))); + "spring.mail.properties.mail.smtp.ssl.enable", "false", + "spring.mail.properties.mail.smtp.starttls.enable", "false"))); var inspection = new ManagedActiveConfigurationInspector.Inspection( ManagedActiveConfigurationInspector.State.LOADABLE, Map.of(), Map.of()); @@ -41,21 +95,21 @@ class SetupStatusProjectionFactoryTest { assertThat(projection.managementDatabase().kind()).isEqualTo(MetadataDatabaseKind.POSTGRESQL); assertThat(projection.managementDatabase().source()).isEqualTo(ConfigSource.SYSTEM_PROPERTY); - assertThat(projection.optional().publicAccessConfigured()).isTrue(); + assertThat(projection.optional().serverOtlpHttpConfigured()).isTrue(); assertThat(projection.optional().retentionConfigured()).isTrue(); assertThat(projection.optional().mailConfigured()).isTrue(); assertThat(projection.warnings()).containsExactlyInAnyOrder( - SetupWarningCode.PUBLIC_ADDRESS_PLAINTEXT, SetupWarningCode.MAIL_SECURITY_NONE); + SetupWarningCode.SERVER_OTLP_PLAINTEXT, SetupWarningCode.MAIL_SECURITY_NONE); } @Test - void unknownExternalMailSecurityDoesNotBreakRestartProjection() { + void incompleteExternalMailSecurityDoesNotBreakRestartProjection() { StandardEnvironment environment = new StandardEnvironment(); environment.getPropertySources().replace( StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME, new MapPropertySource(StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME, Map.of( "spring.jpa.database", "MYSQL", - "hertzbeat.setup.mail.security", "legacy-value"))); + "spring.mail.properties.mail.smtp.ssl.enable", "false"))); var inspection = new ManagedActiveConfigurationInspector.Inspection( ManagedActiveConfigurationInspector.State.ABSENT, Map.of(), Map.of()); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionalConfigurationPersistenceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionalConfigurationPersistenceTest.java index e6f487197a..e9e031ced2 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionalConfigurationPersistenceTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionalConfigurationPersistenceTest.java @@ -18,6 +18,7 @@ package org.apache.hertzbeat.manager.setup.config; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.util.Optional; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; @@ -34,10 +35,10 @@ class ManagedOptionalConfigurationPersistenceTest { ManagedConfigurationTransaction transaction = new ManagedConfigurationTransaction(root); assertThat(transaction.apply(required())).isEqualTo(ManagedConfigurationTransaction.Outcome.APPLIED); ManagedOptionalConfiguration options = new ManagedOptionalConfiguration( - Optional.of(new ManagedOptionalConfiguration.PublicAccessSettings( - Optional.of("https://hertzbeat.example"), - Optional.of("https://hertzbeat.example/otlp"), Optional.of("hertzbeat.example:4317"))), - Optional.of(new ManagedOptionalConfiguration.RetentionSettings(30, 14, 7)), + Optional.of(new ManagedOptionalConfiguration.ServerInstrumentationSettings( + Optional.of("https://hertzbeat.example/otlp"), + Optional.of("https://hertzbeat.example:4317"))), + Optional.of(new ManagedOptionalConfiguration.RetentionSettings(30)), Optional.of(new ManagedOptionalConfiguration.MailSettings( "smtp.example", 465, MailSecurity.TLS, Optional.of("mailer"), "alerts@example.test"))); @@ -50,8 +51,77 @@ class ManagedOptionalConfigurationPersistenceTest { assertThat(application.metadataDatabase()).isEqualTo(required().application().metadataDatabase()); assertThat(application.optional()).isEqualTo(options); assertThat(secrets.mailPassword()).get().isEqualTo(SecretValue.of("mail-secret")); - assertThat(ApplicationConfigDocumentCodec.springProperties(application).toString()) - .doesNotContain("mail-secret"); + var properties = ApplicationConfigDocumentCodec.springProperties(application); + assertThat(properties) + .containsEntry("hertzbeat.instrumentation.server.otlp-http-endpoint", + "https://hertzbeat.example/otlp") + .containsEntry("hertzbeat.instrumentation.server.otlp-grpc-endpoint", + "https://hertzbeat.example:4317") + .containsEntry("hertzbeat.instrumentation.server.profile-id", "server-direct") + .containsEntry("hertzbeat.instrumentation.server.authentication", "bearer_token") + .containsEntry("warehouse.store.greptime.expire-time", "30d") + .containsEntry("spring.mail.properties.mail.smtp.ssl.enable", "true") + .containsEntry("spring.mail.properties.mail.smtp.starttls.enable", "false") + .containsEntry("hertzbeat.mail.from-address", "alerts@example.test") + .doesNotContainKeys("hertzbeat.setup.public-base-url", "hertzbeat.setup.retention.metrics-days", + "hertzbeat.setup.retention.logs-days", "hertzbeat.setup.retention.traces-days", + "hertzbeat.setup.mail.security"); + assertThat(properties.toString()).doesNotContain("mail-secret"); + } + + @Test + void rejectsServerEndpointsWithoutCompleteInternalProfileSettings() throws Exception { + ManagedApplicationConfig application = required().application(); + ManagedOptionalConfiguration options = new ManagedOptionalConfiguration( + Optional.of(new ManagedOptionalConfiguration.ServerInstrumentationSettings( + Optional.of("https://hertzbeat.example/otlp"), Optional.empty())), + Optional.empty(), Optional.empty()); + application = new ManagedApplicationConfig( + application.metadataDatabase(), application.telemetryStore(), options); + ApplicationConfigDocumentCodec codec = new ApplicationConfigDocumentCodec(); + ManagedDocumentCodec.Integrity.VerifiedBody encoded = ManagedDocumentCodec.Integrity.extract( + codec.encode(application, "generation")); + String incomplete = encoded.content().replace( + "hertzbeat.instrumentation.server.authentication: 'bearer_token'\n", ""); + + byte[] document = ManagedDocumentCodec.Integrity.envelope(incomplete, encoded.generation()); + assertThatThrownBy(() -> codec.decode(document)) + .isInstanceOf(ManagedDocumentCodec.DocumentException.class); + } + + @Test + void rejectsBlankManagedServerEndpoint() { + assertThatThrownBy(() -> new ManagedOptionalConfiguration.ServerInstrumentationSettings( + Optional.of(" "), Optional.empty())) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void rejectsControlOnlyManagedServerEndpoint() { + assertThatThrownBy(() -> new ManagedOptionalConfiguration.ServerInstrumentationSettings( + Optional.of("\u0000"), Optional.empty())) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void rejectsBlankServerEndpointInManagedDocument() throws Exception { + ManagedApplicationConfig application = required().application(); + ManagedOptionalConfiguration options = new ManagedOptionalConfiguration( + Optional.of(new ManagedOptionalConfiguration.ServerInstrumentationSettings( + Optional.of("https://hertzbeat.example/otlp"), Optional.empty())), + Optional.empty(), Optional.empty()); + application = new ManagedApplicationConfig( + application.metadataDatabase(), application.telemetryStore(), options); + ApplicationConfigDocumentCodec codec = new ApplicationConfigDocumentCodec(); + ManagedDocumentCodec.Integrity.VerifiedBody encoded = ManagedDocumentCodec.Integrity.extract( + codec.encode(application, "generation")); + String blankEndpoint = encoded.content().replace( + "hertzbeat.instrumentation.server.otlp-http-endpoint: 'https://hertzbeat.example/otlp'", + "hertzbeat.instrumentation.server.otlp-http-endpoint: ' '"); + + byte[] document = ManagedDocumentCodec.Integrity.envelope(blankEndpoint, encoded.generation()); + assertThatThrownBy(() -> codec.decode(document)) + .isInstanceOf(ManagedDocumentCodec.DocumentException.class); } private static ManagedConfigurationBundle required() { diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializerTest.java index 14ec1674ea..b4201e408b 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializerTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializerTest.java @@ -101,12 +101,12 @@ class UnattendedSetupInitializerTest { MockEnvironment environment = new MockEnvironment() .withProperty(UnattendedSetupInitializer.ENABLED_PROPERTY, "true") .withProperty("hertzbeat.setup.unattended.acknowledged-warnings", - "h2_non_production, public_address_plaintext"); + "h2_non_production, server_otlp_plaintext"); new UnattendedSetupInitializer(workflow, environment, new SetupPasswordFileLoader()).initialize(); verify(workflow).complete(java.util.List.of( - SetupWarningCode.H2_NON_PRODUCTION, SetupWarningCode.PUBLIC_ADDRESS_PLAINTEXT)); + SetupWarningCode.H2_NON_PRODUCTION, SetupWarningCode.SERVER_OTLP_PLAINTEXT)); } private static StatusResponse status(SetupPhase phase) { @@ -115,6 +115,6 @@ class UnattendedSetupInitializerTest { new ManagementDatabaseSummary(MetadataDatabaseKind.H2, true, ConfigSource.UI_MANAGED, false), new TelemetryStoreSummary(TelemetryStoreKind.GREPTIME, true, ConfigSource.UI_MANAGED, false), phase != SetupPhase.ADMINISTRATOR_REQUIRED, - new OptionalConfigurationSummary(false, false, false, false, false)); + new OptionalConfigurationSummary(false, false, false, false)); } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java index c7daabd8fe..8cf8129d6a 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java @@ -17,7 +17,9 @@ package org.apache.hertzbeat.manager.setup.workflow; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; @@ -44,7 +46,7 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationResp import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsRequest; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.PublicAccessConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ServerInstrumentationConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupAccess; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; @@ -62,6 +64,31 @@ import org.junit.jupiter.api.Test; class DefaultSetupWorkflowTest { + @Test + void optionSummaryUsesMeaningfulEndpointSemantics() { + ManagedConfigCapability capability = mock(ManagedConfigCapability.class); + SetupRuntimeState state = new SetupRuntimeState(Clock.systemUTC(), capability, + SetupPhase.OPTIONAL_CONFIGURATION, SetupAccess.LOCAL, true, "operator"); + SetupRequestValidator validator = mock(SetupRequestValidator.class); + when(validator.validate(any(ValidateRequest.class))) + .thenReturn(new ValidationResponse(true, Instant.now(), null, List.of())); + DefaultSetupWorkflow workflow = workflow(state, validator, + mock(SetupConfigurationCoordinator.class), mock(SetupOperationRegistry.class), capability, + Optional.of(mock(IdentityInitializationService.class)), + Optional.of(mock(SetupCompletionCoordinator.class)), mock(SetupOptionsCoordinator.class), + Clock.systemUTC(), new SetupMutationSerializer()); + OptionsRequest request = new OptionsRequest( + new ServerInstrumentationConfiguration("https://server.example.test:4318", "\u0000"), + null, null); + + var response = workflow.configureOptions(request); + + assertTrue(response.serverOtlpHttpConfigured()); + assertFalse(response.serverOtlpGrpcConfigured()); + assertTrue(state.status().optional().serverOtlpHttpConfigured()); + assertFalse(state.status().optional().serverOtlpGrpcConfigured()); + } + @Test void wrongPhaseMustBeRejectedBeforeAdministratorOrCompletionWrites() { ManagedConfigCapability capability = mock(ManagedConfigCapability.class); @@ -130,7 +157,7 @@ class DefaultSetupWorkflowTest { mock(SetupConfigurationCoordinator.class), capability, Optional.of(mock(IdentityInitializationService.class)), Optional.of(completion), mutations); OptionsRequest request = new OptionsRequest( - new PublicAccessConfiguration("http://localhost:1157", null, null), null, null); + new ServerInstrumentationConfiguration("http://localhost:4318", null), null, null); try (var executor = Executors.newFixedThreadPool(2)) { var optionsResult = executor.submit(() -> workflow.configureOptions(request)); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinatorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinatorTest.java index 92a3863c22..aecb2135c6 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinatorTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinatorTest.java @@ -40,9 +40,9 @@ class HeadlessSetupCoordinatorTest { ManagedConfigCapability capability = mock(ManagedConfigCapability.class); SetupRuntimeState state = new SetupRuntimeState(Clock.systemUTC(), capability, SetupPhase.OPTIONAL_CONFIGURATION, SetupAccess.LOCAL, true, "operator"); - state.optionsConfigured(new OptionalConfigurationSummary(true, false, false, false, false), + state.optionsConfigured(new OptionalConfigurationSummary(true, false, false, false), List.of(SetupWarningCode.H2_NON_PRODUCTION, - SetupWarningCode.PUBLIC_ADDRESS_PLAINTEXT)); + SetupWarningCode.SERVER_OTLP_PLAINTEXT)); SetupCompletionCoordinator completion = mock(SetupCompletionCoordinator.class); SetupTransitionService transitions = new SetupTransitionService(state, mock(SetupRequestValidator.class), mock(SetupConfigurationCoordinator.class), capability, @@ -55,7 +55,7 @@ class HeadlessSetupCoordinatorTest { verifyNoInteractions(completion); coordinator.complete(List.of(SetupWarningCode.H2_NON_PRODUCTION, - SetupWarningCode.PUBLIC_ADDRESS_PLAINTEXT)); + SetupWarningCode.SERVER_OTLP_PLAINTEXT)); verify(completion).completeInstallation(); } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidatorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidatorTest.java index c50ac3d549..4bb9425845 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidatorTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidatorTest.java @@ -26,8 +26,9 @@ import java.time.Instant; import java.time.ZoneOffset; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.PublicAccessConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ServerInstrumentationConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidateRequest; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationSection; import org.junit.jupiter.api.Test; @@ -49,20 +50,60 @@ class SetupRequestValidatorTest { } @Test - void publicAddressValidatorProducesStablePlaintextWarning() { - var response = validator.validate(new ValidateRequest(ValidationSection.PUBLIC_ACCESS, - null, null, new PublicAccessConfiguration("http://monitor.example.test", null, null), null)); + void serverInstrumentationValidatorProducesStablePlaintextWarning() { + var response = validator.validate(new ValidateRequest(ValidationSection.SERVER_INSTRUMENTATION, + null, null, new ServerInstrumentationConfiguration("http://monitor.example.test", null), null)); assertTrue(response.valid()); assertEquals(1, response.warnings().size()); } @Test - void publicGrpcPortMustFitTheTransportRange() { - var response = validator.validate(new ValidateRequest(ValidationSection.PUBLIC_ACCESS, - null, null, new PublicAccessConfiguration(null, null, "collector.example.test:99999"), null)); + void serverGrpcEndpointMustBeAnExplicitHttpUrl() { + var response = validator.validate(new ValidateRequest(ValidationSection.SERVER_INSTRUMENTATION, + null, null, new ServerInstrumentationConfiguration(null, "collector.example.test:4317"), null)); assertFalse(response.valid()); - assertEquals(SetupErrorCode.PUBLIC_ADDRESS_INVALID, response.errorCode()); + assertEquals(SetupErrorCode.SERVER_INSTRUMENTATION_INVALID, response.errorCode()); + } + + @Test + void serverEndpointRejectsUrlCredentialsAndQuery() { + var response = validator.validate(new ValidateRequest(ValidationSection.SERVER_INSTRUMENTATION, + null, null, new ServerInstrumentationConfiguration( + "https://user:secret@collector.example.test:4318?token=secret", null), null)); + + assertFalse(response.valid()); + assertEquals(SetupErrorCode.SERVER_INSTRUMENTATION_INVALID, response.errorCode()); + } + + @Test + void grpcOnlyPlaintextEndpointProducesWarning() { + var response = validator.validate(new ValidateRequest(ValidationSection.SERVER_INSTRUMENTATION, + null, null, new ServerInstrumentationConfiguration(null, "http://collector.example.test:4317"), + null)); + + assertTrue(response.valid()); + assertEquals(java.util.List.of(SetupWarningCode.SERVER_OTLP_PLAINTEXT), response.warnings()); + } + + @Test + void serverInstrumentationSectionRequiresAtLeastOneEndpoint() { + var response = validator.validate(new ValidateRequest(ValidationSection.SERVER_INSTRUMENTATION, + null, null, new ServerInstrumentationConfiguration(" ", null), null)); + + assertFalse(response.valid()); + assertEquals(SetupErrorCode.SERVER_INSTRUMENTATION_INVALID, response.errorCode()); + } + + @Test + void endpointWhitespaceIsNormalizedBeforeValidationAndWarnings() { + var response = validator.validate(new ValidateRequest(ValidationSection.SERVER_INSTRUMENTATION, + null, null, new ServerInstrumentationConfiguration( + " https://collector.example.test:4318/otlp ", + " http://collector.example.test:4317 "), null)); + + assertTrue(response.valid()); + assertEquals(java.util.List.of(SetupWarningCode.SERVER_OTLP_PLAINTEXT), response.warnings()); } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupWarningPolicyTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupWarningPolicyTest.java index 80d20cec86..c79687f94c 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupWarningPolicyTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupWarningPolicyTest.java @@ -23,17 +23,27 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailConfiguration import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsRequest; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.PublicAccessConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ServerInstrumentationConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; import org.junit.jupiter.api.Test; class SetupWarningPolicyTest { @Test void liveAndRestartInputsProduceTheSameWarnings() { var options = new OptionsRequest( - new PublicAccessConfiguration("http://localhost:1157", null, null), null, + new ServerInstrumentationConfiguration("http://localhost:4318", null), null, new MailConfiguration("localhost", 25, MailSecurity.NONE, null, null, "ops@example.test")); assertThat(SetupWarningPolicy.INSTANCE.evaluate(MetadataDatabaseKind.H2, options)) .containsExactlyElementsOf(SetupWarningPolicy.INSTANCE.evaluate( - MetadataDatabaseKind.H2, "http://localhost:1157", MailSecurity.NONE)); + MetadataDatabaseKind.H2, "http://localhost:4318", null, MailSecurity.NONE)); + } + + @Test + void whitespaceWrappedGrpcEndpointStillProducesPlaintextWarning() { + var options = new OptionsRequest( + new ServerInstrumentationConfiguration(null, " http://localhost:4317 "), null, null); + + assertThat(SetupWarningPolicy.INSTANCE.evaluate(MetadataDatabaseKind.MYSQL, options)) + .containsExactly(SetupWarningCode.SERVER_OTLP_PLAINTEXT); } } diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/config/ManagedConfigDataPrecedenceTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/config/ManagedConfigDataPrecedenceTest.java index 65aa60f5bb..495edfd42d 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/config/ManagedConfigDataPrecedenceTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/config/ManagedConfigDataPrecedenceTest.java @@ -19,12 +19,17 @@ package org.apache.hertzbeat.startup.config; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.nio.file.Files; import java.nio.file.Path; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Optional; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigSource; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; import org.apache.hertzbeat.manager.setup.config.EffectiveConfigurationResolver; import org.apache.hertzbeat.manager.setup.config.EffectiveConfigurationValue; @@ -34,14 +39,26 @@ import org.apache.hertzbeat.manager.setup.config.ManagedApplicationConfig; import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector; import org.apache.hertzbeat.manager.setup.config.ManagedConfigurationBundle; import org.apache.hertzbeat.manager.setup.config.ManagedConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.ManagedOptionalConfiguration; import org.apache.hertzbeat.manager.setup.config.ManagedSecrets; import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; import org.apache.hertzbeat.manager.setup.config.RestartRequirement; import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.apache.hertzbeat.manager.dao.CollectorDao; +import org.apache.hertzbeat.manager.instrumentation.intake.CollectorIntakeAdvertisementReader; +import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.Authentication; +import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.Availability; +import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.Gateway; +import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.OtlpTransport; +import org.apache.hertzbeat.startup.instrumentation.ExternalOtelCollectorIntakeProperties; +import org.apache.hertzbeat.startup.instrumentation.ManagerInstrumentationIntakeProfileStore; +import org.apache.hertzbeat.startup.instrumentation.ServerInstrumentationIntakeProperties; +import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.springframework.boot.SpringApplication; import org.springframework.boot.WebApplicationType; +import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Configuration; import org.springframework.core.env.ConfigurableEnvironment; @@ -49,6 +66,8 @@ import org.springframework.core.env.MapPropertySource; import org.springframework.core.env.PropertySource; import org.springframework.core.env.StandardEnvironment; import org.springframework.core.env.SystemEnvironmentPropertySource; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; class ManagedConfigDataPrecedenceTest { @@ -98,6 +117,49 @@ class ManagedConfigDataPrecedenceTest { new String[] {"--" + KEY + "=cli"}, "cli", "commandLineArgs", ConfigSource.COMMAND_LINE); } + @Test + void managedOptionalSettingsReachRuntimeConsumers() throws Exception { + Path installationRoot = Files.createDirectories(temporaryDirectory.resolve("runtime-consumers")); + ManagedSecrets managedSecrets = new ManagedSecrets(SecretValue.of(TEST_PASSWORD), Optional.empty(), + Optional.of(SecretValue.of("mail-secret"))); + ManagedConfigurationTransaction transaction = new ManagedConfigurationTransaction(installationRoot); + assertEquals(ManagedConfigurationTransaction.Outcome.APPLIED, + transaction.apply(new ManagedConfigurationBundle(managedApplicationWithOptions(), managedSecrets))); + + ConfigurableEnvironment environment = new StandardEnvironment(); + environment.getPropertySources().addFirst(new MapPropertySource("testInstallationRoot", + Map.of(INSTALLATION_ROOT, installationRoot.toString()))); + SpringApplication application = new SpringApplication(RuntimeConsumerBinding.class); + application.setEnvironment(environment); + application.setWebApplicationType(WebApplicationType.NONE); + application.setLogStartupInfo(false); + try (ConfigurableApplicationContext context = application.run()) { + ServerInstrumentationIntakeProperties server = + context.getBean(ServerInstrumentationIntakeProperties.class); + assertEquals("http://server.example.test:4318", server.otlpHttpEndpoint()); + assertEquals("https://server.example.test:4317", server.otlpGrpcEndpoint()); + CollectorDao collectorDao = mock(CollectorDao.class); + when(collectorDao.findAll(any(Pageable.class))).thenReturn(Page.empty()); + var profile = new ManagerInstrumentationIntakeProfileStore( + collectorDao, mock(CollectorIntakeAdvertisementReader.class), server, + new ExternalOtelCollectorIntakeProperties(null, null, null, null)) + .profiles().getFirst(); + assertEquals("server-direct", profile.id()); + assertEquals(Availability.AVAILABLE, profile.availability()); + assertEquals(Gateway.SERVER, profile.gateway()); + assertEquals(Authentication.BEARER_TOKEN, profile.authentication()); + assertEquals(java.util.List.of(OtlpTransport.HTTP_PROTOBUF, OtlpTransport.GRPC), + profile.supportedTransports()); + assertEquals("30d", context.getBean(GreptimeProperties.class).expireTime()); + assertEquals("true", context.getEnvironment().getProperty( + "spring.mail.properties.mail.smtp.ssl.enable")); + assertEquals("false", context.getEnvironment().getProperty( + "spring.mail.properties.mail.smtp.starttls.enable")); + assertEquals("alerts@example.test", + context.getEnvironment().getProperty("hertzbeat.mail.from-address")); + } + } + private static void assertLayer( Map systemProperties, Map environmentVariables, @@ -160,7 +222,24 @@ class ManagedConfigDataPrecedenceTest { new GreptimeEndpoints("greptime:4001", "http://greptime:4000"), "public")); } + private static ManagedApplicationConfig managedApplicationWithOptions() { + ManagedOptionalConfiguration options = new ManagedOptionalConfiguration( + Optional.of(new ManagedOptionalConfiguration.ServerInstrumentationSettings( + Optional.of("http://server.example.test:4318"), + Optional.of("https://server.example.test:4317"))), + Optional.of(new ManagedOptionalConfiguration.RetentionSettings(30)), + Optional.of(new ManagedOptionalConfiguration.MailSettings("smtp.example.test", 465, + MailSecurity.TLS, Optional.of("mailer@example.test"), "alerts@example.test"))); + ManagedApplicationConfig required = managedApplication(); + return new ManagedApplicationConfig(required.metadataDatabase(), required.telemetryStore(), options); + } + @Configuration(proxyBeanMethods = false) static class ProbeConfiguration { } + + @Configuration(proxyBeanMethods = false) + @EnableConfigurationProperties({ServerInstrumentationIntakeProperties.class, GreptimeProperties.class}) + static class RuntimeConsumerBinding { + } } From 1e0bcc6547ec58dd29b6f889d81c8024dccc3400 Mon Sep 17 00:00:00 2001 From: Logic Date: Sun, 9 Aug 2026 02:23:39 +0800 Subject: [PATCH 16/71] Preserve setup operations across startup --- .../setup/api/SetupApiConfiguration.java | 4 +- .../SetupOperationCheckpointStore.java | 118 ++++++++++++++ .../workflow/SetupOperationRegistry.java | 63 +++++++- .../SetupOperationCheckpointStoreTest.java | 60 +++++++ .../SetupOperationRegistryCheckpointTest.java | 146 ++++++++++++++++++ .../runtime/HertzBeatStartupCoordinator.java | 8 +- .../LocalInstallationStartupProbe.java | 49 +++--- .../runtime/StartupArgumentProperties.java | 36 +++++ .../startup/runtime/StartupDecision.java | 10 +- .../startup/runtime/StartupDecisionProbe.java | 2 +- .../runtime/StartupModePropertyProbe.java | 24 +-- .../HertzBeatStartupCoordinatorTest.java | 74 +++++++-- .../LocalInstallationStartupProbeTest.java | 32 +++- .../ManagedConfigRecoveryContextTest.java | 6 +- .../StartupArgumentPropertiesTest.java | 69 +++++++++ .../runtime/StartupModePropertyProbeTest.java | 18 +-- .../StartupRuntimeBoundaryContextTest.java | 65 +++++++- 17 files changed, 697 insertions(+), 87 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOperationCheckpointStore.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupOperationCheckpointStoreTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupOperationRegistryCheckpointTest.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupArgumentProperties.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupArgumentPropertiesTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java index 4a880fd415..4b5cf86e92 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java @@ -73,8 +73,8 @@ public class SetupApiConfiguration { } @Bean - public SetupOperationRegistry setupOperationRegistry() { - return new SetupOperationRegistry(Clock.systemUTC()); + public SetupOperationRegistry setupOperationRegistry(Environment environment, SetupRuntimeState state) { + return new SetupOperationRegistry(Clock.systemUTC(), SetupInstallationPaths.root(environment), state.phase()); } @Bean diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOperationCheckpointStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOperationCheckpointStore.java new file mode 100644 index 0000000000..be3b079970 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOperationCheckpointStore.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.IOException; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.time.Instant; +import java.util.Optional; +import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Persists the one secret-free operation needed to bridge a setup context restart. */ +final class SetupOperationCheckpointStore { + static final String RELATIVE_PATH = "data/config/setup-operation.properties"; + private static final Logger LOGGER = LoggerFactory.getLogger(SetupOperationCheckpointStore.class); + private final Path checkpoint; + + SetupOperationCheckpointStore(Path installationRoot) { + checkpoint = installationRoot.resolve(RELATIVE_PATH); + } + + Optional load() { + try { + if (!Files.exists(checkpoint, LinkOption.NOFOLLOW_LINKS)) { + return Optional.empty(); + } + if (!Files.isRegularFile(checkpoint, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("checkpoint is not a regular file"); + } + Properties properties = new Properties(); + try (var input = Files.newInputStream(checkpoint)) { + properties.load(input); + } + String operationId = properties.getProperty("operationId"); + if (operationId == null || operationId.isBlank()) { + throw new IllegalArgumentException("operationId is missing"); + } + return Optional.of(new Checkpoint(operationId, + Instant.parse(properties.getProperty("createdAt")))); + } catch (IOException | RuntimeException failure) { + LOGGER.warn("Ignoring unreadable or malformed setup operation checkpoint at {}", + checkpoint, failure); + return Optional.empty(); + } + } + + void save(String operationId, Instant createdAt) { + Path temporary = null; + try { + Files.createDirectories(checkpoint.getParent()); + temporary = Files.createTempFile(checkpoint.getParent(), ".setup-operation-", ".tmp"); + Properties properties = new Properties(); + properties.setProperty("operationId", operationId); + properties.setProperty("createdAt", createdAt.toString()); + try (var output = Files.newOutputStream(temporary)) { + properties.store(output, "Secret-free setup operation checkpoint"); + } + move(temporary); + } catch (IOException | RuntimeException failure) { + // Configuration is already durable; checkpoint failure must not block the context transition. + LOGGER.warn("Cannot persist setup operation checkpoint at {}; restart polling may require status refresh", + checkpoint, failure); + } finally { + deleteTemporary(temporary); + } + } + + void delete() { + try { + Files.deleteIfExists(checkpoint); + } catch (IOException | RuntimeException failure) { + LOGGER.warn("Cannot remove consumed setup operation checkpoint at {}", checkpoint, failure); + } + } + + private void move(Path temporary) throws IOException { + try { + Files.move(temporary, checkpoint, StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } catch (AtomicMoveNotSupportedException unsupported) { + Files.move(temporary, checkpoint, StandardCopyOption.REPLACE_EXISTING); + } + } + + private void deleteTemporary(Path temporary) { + if (temporary == null) { + return; + } + try { + Files.deleteIfExists(temporary); + } catch (IOException | RuntimeException failure) { + LOGGER.warn("Cannot remove temporary setup operation checkpoint at {}", temporary, failure); + } + } + + record Checkpoint(String operationId, Instant createdAt) { + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOperationRegistry.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOperationRegistry.java index a222cb4f5a..c4075e4af2 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOperationRegistry.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOperationRegistry.java @@ -17,6 +17,7 @@ package org.apache.hertzbeat.manager.setup.workflow; +import java.nio.file.Path; import java.time.Clock; import java.time.Instant; import java.util.LinkedHashMap; @@ -32,11 +33,21 @@ public final class SetupOperationRegistry { private static final int MAX_HISTORY = 64; private static final long POLL_AFTER_MILLIS = 1_000; private final Clock clock; + private final SetupOperationCheckpointStore checkpointStore; + private final SetupPhase recoveredPhase; private final Map operations = new LinkedHashMap<>(); private String activeOperationId; + private String restoredCheckpointOperationId; public SetupOperationRegistry(Clock clock) { + this(clock, null, null); + } + + public SetupOperationRegistry(Clock clock, Path installationRoot, SetupPhase recoveredPhase) { this.clock = clock; + checkpointStore = installationRoot == null ? null : new SetupOperationCheckpointStore(installationRoot); + this.recoveredPhase = recoveredPhase; + restoreCheckpoint(); } public synchronized String begin(SetupPhase phase) { @@ -69,6 +80,9 @@ public final class SetupOperationRegistry { current.startedAt(), completedAt, errorCode, terminal(state) ? 0 : POLL_AFTER_MILLIS, exportAvailable); operations.put(id, updated); + if (state == SetupOperationState.AWAITING_RESTART) { + saveCheckpoint(updated); + } if (terminal(state)) { activeOperationId = null; } @@ -76,7 +90,13 @@ public final class SetupOperationRegistry { } public synchronized OperationResponse get(String id) { - return operations.get(id); + OperationResponse operation = operations.get(id); + if (operation != null && id.equals(restoredCheckpointOperationId) && terminal(operation.state())) { + // A terminal bridge is consumed only after the rebuilt context exposes it to the caller. + checkpointStore.delete(); + restoredCheckpointOperationId = null; + } + return operation; } private OperationResponse require(String id) { @@ -101,4 +121,45 @@ public final class SetupOperationRegistry { return state == SetupOperationState.SUCCEEDED || state == SetupOperationState.FAILED || state == SetupOperationState.ROLLED_BACK; } + + private void restoreCheckpoint() { + if (checkpointStore == null || recoveredPhase == null) { + return; + } + checkpointStore.load().ifPresent(checkpoint -> { + OperationResponse restored = recoveredOperation(checkpoint.operationId(), checkpoint.createdAt()); + operations.put(checkpoint.operationId(), restored); + restoredCheckpointOperationId = checkpoint.operationId(); + if (!terminal(restored.state())) { + activeOperationId = checkpoint.operationId(); + } + }); + } + + private OperationResponse recoveredOperation(String id, Instant createdAt) { + Instant now = clock.instant(); + if (recoveredPhase == SetupPhase.RECOVERY_REQUIRED) { + return new OperationResponse(id, SetupOperationState.FAILED, recoveredPhase, + createdAt, createdAt, now, SetupErrorCode.CONFIG_RECOVERY_REQUIRED, 0, false); + } + if (recoveredPhase == SetupPhase.ADMINISTRATOR_REQUIRED + || recoveredPhase == SetupPhase.OPTIONAL_CONFIGURATION + || recoveredPhase == SetupPhase.COMPLETE) { + return new OperationResponse(id, SetupOperationState.SUCCEEDED, recoveredPhase, + createdAt, createdAt, now, null, 0, false); + } + if (recoveredPhase == SetupPhase.APPLICATION_STARTING) { + return new OperationResponse(id, SetupOperationState.AWAITING_RESTART, + recoveredPhase, createdAt, createdAt, null, null, POLL_AFTER_MILLIS, false); + } + // A rebuilt context asking for configuration again proves the prior apply did not converge. + return new OperationResponse(id, SetupOperationState.ROLLED_BACK, recoveredPhase, + createdAt, createdAt, now, SetupErrorCode.CONFIG_WRITE_FAILED, 0, false); + } + + private void saveCheckpoint(OperationResponse operation) { + if (checkpointStore != null) { + checkpointStore.save(operation.operationId(), operation.createdAt()); + } + } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupOperationCheckpointStoreTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupOperationCheckpointStoreTest.java new file mode 100644 index 0000000000..6878fee8c0 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupOperationCheckpointStoreTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class SetupOperationCheckpointStoreTest { + @TempDir + private Path root; + + @Test + void savesAndLoadsSecretFreeCheckpoint() { + SetupOperationCheckpointStore store = new SetupOperationCheckpointStore(root); + Instant createdAt = Instant.parse("2026-08-09T00:00:00Z"); + + store.save("operation-1", createdAt); + + var checkpoint = store.load().orElseThrow(); + assertEquals("operation-1", checkpoint.operationId()); + assertEquals(createdAt, checkpoint.createdAt()); + } + + @Test + void malformedCheckpointIsIgnoredWithoutBecomingApiState() throws Exception { + Path checkpoint = root.resolve(SetupOperationCheckpointStore.RELATIVE_PATH); + Files.createDirectories(checkpoint.getParent()); + Files.writeString(checkpoint, "operationId=operation-1\ncreatedAt=not-an-instant\n"); + + assertTrue(new SetupOperationCheckpointStore(root).load().isEmpty()); + } + + @Test + void unreadableCheckpointShapeIsIgnoredWithoutBecomingApiState() throws Exception { + Files.createDirectories(root.resolve(SetupOperationCheckpointStore.RELATIVE_PATH)); + + assertTrue(new SetupOperationCheckpointStore(root).load().isEmpty()); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupOperationRegistryCheckpointTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupOperationRegistryCheckpointTest.java new file mode 100644 index 0000000000..5af1142956 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupOperationRegistryCheckpointTest.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class SetupOperationRegistryCheckpointTest { + private static final Instant NOW = Instant.parse("2026-08-09T00:00:00Z"); + private static final Clock CLOCK = Clock.fixed(NOW, ZoneOffset.UTC); + + @TempDir + private Path root; + + @Test + void successfulRestoreProjectsTerminalStateAndConsumesCheckpoint() { + String operationId = awaitingRestart(root); + Path checkpoint = checkpoint(root); + + SetupOperationRegistry restored = new SetupOperationRegistry( + CLOCK, root, SetupPhase.ADMINISTRATOR_REQUIRED); + + assertTrue(Files.exists(checkpoint)); + var response = restored.get(operationId); + assertNotNull(response); + assertEquals(SetupOperationState.SUCCEEDED, response.state()); + assertEquals(SetupPhase.ADMINISTRATOR_REQUIRED, response.phase()); + assertFalse(Files.exists(checkpoint)); + } + + @Test + void recoveryRestoreProjectsFailedAndConsumesCheckpoint() { + String operationId = awaitingRestart(root); + + SetupOperationRegistry restored = new SetupOperationRegistry( + CLOCK, root, SetupPhase.RECOVERY_REQUIRED); + + assertTrue(Files.exists(checkpoint(root))); + var response = restored.get(operationId); + assertNotNull(response); + assertEquals(SetupOperationState.FAILED, response.state()); + assertEquals(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, response.errorCode()); + assertFalse(Files.exists(checkpoint(root))); + } + + @Test + void checkpointWriteFailureDoesNotAbortAwaitingRestartResponse() throws Exception { + Files.writeString(root.resolve("data"), "parent-is-not-a-directory"); + SetupOperationRegistry registry = new SetupOperationRegistry( + CLOCK, root, SetupPhase.CONFIGURATION_REQUIRED); + String operationId = registry.begin(SetupPhase.CONFIGURATION_REQUIRED); + + var response = assertDoesNotThrow(() -> registry.finish(operationId, + SetupOperationState.AWAITING_RESTART, SetupPhase.APPLICATION_STARTING, null, false)); + + assertEquals(SetupOperationState.AWAITING_RESTART, response.state()); + assertEquals(response, registry.get(operationId)); + } + + @Test + void configurationRequiredRestoreRollsBackConsumesCheckpointAndAllowsRepair() { + String operationId = awaitingRestart(root); + SetupOperationRegistry restored = new SetupOperationRegistry( + CLOCK, root, SetupPhase.CONFIGURATION_REQUIRED); + + assertTrue(Files.exists(checkpoint(root))); + var response = restored.get(operationId); + + assertNotNull(response); + assertEquals(SetupOperationState.ROLLED_BACK, response.state()); + assertEquals(SetupErrorCode.CONFIG_WRITE_FAILED, response.errorCode()); + assertFalse(Files.exists(checkpoint(root))); + assertNotNull(restored.begin(SetupPhase.CONFIGURATION_REQUIRED)); + } + + @Test + void applicationStartingRestoreRemainsAwaitingAndActive() { + String operationId = awaitingRestart(root); + SetupOperationRegistry restored = new SetupOperationRegistry( + CLOCK, root, SetupPhase.APPLICATION_STARTING); + + assertEquals(SetupOperationState.AWAITING_RESTART, restored.get(operationId).state()); + assertThrows(SetupWorkflowConflict.class, + () -> restored.begin(SetupPhase.CONFIGURATION_REQUIRED)); + assertTrue(Files.exists(checkpoint(root))); + } + + @Test + void unobservedSuccessCanBeReprojectedAsFailedByRecoveryContext() { + String operationId = awaitingRestart(root); + new SetupOperationRegistry(CLOCK, root, SetupPhase.ADMINISTRATOR_REQUIRED); + assertTrue(Files.exists(checkpoint(root))); + + SetupOperationRegistry recovery = new SetupOperationRegistry( + CLOCK, root, SetupPhase.RECOVERY_REQUIRED); + + assertTrue(Files.exists(checkpoint(root))); + var response = recovery.get(operationId); + assertEquals(SetupOperationState.FAILED, response.state()); + assertEquals(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, response.errorCode()); + assertFalse(Files.exists(checkpoint(root))); + } + + private static String awaitingRestart(Path root) { + SetupOperationRegistry registry = new SetupOperationRegistry( + CLOCK, root, SetupPhase.CONFIGURATION_REQUIRED); + String operationId = registry.begin(SetupPhase.CONFIGURATION_REQUIRED); + registry.finish(operationId, SetupOperationState.AWAITING_RESTART, + SetupPhase.APPLICATION_STARTING, null, false); + return operationId; + } + + private static Path checkpoint(Path root) { + return root.resolve(SetupOperationCheckpointStore.RELATIVE_PATH); + } +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java index d6a3569201..29766e312c 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java @@ -19,7 +19,6 @@ package org.apache.hertzbeat.startup.runtime; import java.util.Objects; import org.apache.hertzbeat.common.runtime.RuntimeMode; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; /** Serializes setup-to-normal transitions and always closes the old context first. */ @@ -39,7 +38,7 @@ public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition args = applicationArgs == null ? new String[0] : applicationArgs.clone(); StartupDecision decision; try { - decision = Objects.requireNonNull(probe.probe(), "startup decision"); + decision = Objects.requireNonNull(probe.probe(args.clone()), "startup decision"); } catch (RuntimeException exception) { decision = StartupDecision.recovery(); } @@ -48,13 +47,12 @@ public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition @Override public synchronized void configurationApplied() { - transition(new StartupDecision(RuntimeMode.FULL_SETUP_GATED, - SetupPhase.ADMINISTRATOR_REQUIRED, null)); + transition(new StartupDecision(RuntimeMode.FULL_SETUP_GATED)); } @Override public synchronized void completeSetup() { - transition(new StartupDecision(RuntimeMode.NORMAL, SetupPhase.COMPLETE, null)); + transition(StartupDecision.normal()); } public synchronized RunningApplicationContext transition(StartupDecision decision) { diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbe.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbe.java index 54f170efe4..bc561bc9d0 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbe.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbe.java @@ -23,7 +23,6 @@ import java.nio.file.LinkOption; import java.nio.file.Path; import java.security.SecureRandom; import org.apache.hertzbeat.common.runtime.RuntimeMode; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector; import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector.State; import org.apache.hertzbeat.manager.setup.config.SetupInstallationPaths; @@ -31,43 +30,47 @@ import org.apache.hertzbeat.manager.setup.installation.LocalInstallationFingerpr /** Filesystem-first startup convergence with an explicit legacy/external upgrade entry. */ public final class LocalInstallationStartupProbe implements StartupDecisionProbe { - private final Path root; - private final boolean externalDatabaseConfigured; + private static final String DATASOURCE_PROPERTY = "spring.datasource.url"; + private static final String DATASOURCE_ENVIRONMENT = "SPRING_DATASOURCE_URL"; + private static final String ROOT_ENVIRONMENT = "HERTZBEAT_INTERNAL_INSTALLATION_ROOT"; + private final Path fixedRoot; + private final Boolean fixedExternalDatabaseConfigured; public LocalInstallationStartupProbe() { - this(Path.of(System.getProperty(SetupInstallationPaths.ROOT_PROPERTY, ".")), - externalDatabaseConfigured()); + fixedRoot = null; + fixedExternalDatabaseConfigured = null; } LocalInstallationStartupProbe(Path root, boolean externalDatabaseConfigured) { - this.root = root.toAbsolutePath().normalize(); - this.externalDatabaseConfigured = externalDatabaseConfigured; + fixedRoot = root.toAbsolutePath().normalize(); + fixedExternalDatabaseConfigured = externalDatabaseConfigured; } @Override - public StartupDecision probe() { + public StartupDecision probe(String[] args) { + Path root = fixedRoot == null ? installationRoot(args) : fixedRoot; + boolean externalDatabaseConfigured = fixedExternalDatabaseConfigured == null + ? externalDatabaseConfigured(args) : fixedExternalDatabaseConfigured; State managed = new ManagedActiveConfigurationInspector(root).inspect().state(); if (managed == State.RECOVERY_REQUIRED) { return StartupDecision.recovery(); } - FingerprintState fingerprint = fingerprintState(); + FingerprintState fingerprint = fingerprintState(root); if (fingerprint == FingerprintState.INVALID) { return StartupDecision.recovery(); } - boolean legacyDatabase = legacyH2Present(); + boolean legacyDatabase = legacyH2Present(root); if (fingerprint == FingerprintState.PRESENT) { return managed == State.LOADABLE || legacyDatabase || externalDatabaseConfigured - ? new StartupDecision(RuntimeMode.FULL_SETUP_GATED, - SetupPhase.ADMINISTRATOR_REQUIRED, null) : StartupDecision.recovery(); + ? new StartupDecision(RuntimeMode.FULL_SETUP_GATED) : StartupDecision.recovery(); } if (managed == State.LOADABLE || legacyDatabase || externalDatabaseConfigured) { - return new StartupDecision(RuntimeMode.FULL_SETUP_GATED, - SetupPhase.ADMINISTRATOR_REQUIRED, null); + return new StartupDecision(RuntimeMode.FULL_SETUP_GATED); } - return new StartupDecision(RuntimeMode.SETUP_ONLY, SetupPhase.CONFIGURATION_REQUIRED, null); + return new StartupDecision(RuntimeMode.SETUP_ONLY); } - private FingerprintState fingerprintState() { + private static FingerprintState fingerprintState(Path root) { Path path = root.resolve("data/config/.installation-fingerprint"); try { boolean present = new LocalInstallationFingerprintStore(path, new SecureRandom()).read().isPresent(); @@ -81,14 +84,20 @@ public final class LocalInstallationStartupProbe implements StartupDecisionProbe } } - private boolean legacyH2Present() { + private static boolean legacyH2Present(Path root) { return Files.isRegularFile(root.resolve("data/hertzbeat.mv.db")) || Files.isRegularFile(root.resolve("data/hertzbeat.h2.db")); } - private static boolean externalDatabaseConfigured() { - return hasText(System.getProperty("spring.datasource.url")) - || hasText(System.getenv("SPRING_DATASOURCE_URL")); + private static Path installationRoot(String[] args) { + String configured = StartupArgumentProperties.resolve(args, SetupInstallationPaths.ROOT_PROPERTY, + System.getProperty(SetupInstallationPaths.ROOT_PROPERTY), System.getenv(ROOT_ENVIRONMENT)); + return Path.of(configured == null ? "." : configured).toAbsolutePath().normalize(); + } + + private static boolean externalDatabaseConfigured(String[] args) { + return hasText(StartupArgumentProperties.resolve(args, DATASOURCE_PROPERTY, + System.getProperty(DATASOURCE_PROPERTY), System.getenv(DATASOURCE_ENVIRONMENT))); } private static boolean hasText(String value) { diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupArgumentProperties.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupArgumentProperties.java new file mode 100644 index 0000000000..de4deb2b1f --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupArgumentProperties.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup.runtime; + +import org.springframework.core.env.SimpleCommandLinePropertySource; + +/** Resolves explicitly allowed pre-Spring properties without exposing arbitrary application arguments. */ +final class StartupArgumentProperties { + + private StartupArgumentProperties() { + } + + static String resolve(String[] args, String propertyName, String systemValue, String environmentValue) { + String commandLineValue = new SimpleCommandLinePropertySource( + args == null ? new String[0] : args).getProperty(propertyName); + if (commandLineValue != null) { + return commandLineValue; + } + return systemValue == null ? environmentValue : systemValue; + } +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupDecision.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupDecision.java index 7669714a1e..adc691a01c 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupDecision.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupDecision.java @@ -19,23 +19,19 @@ package org.apache.hertzbeat.startup.runtime; import java.util.Objects; import org.apache.hertzbeat.common.runtime.RuntimeMode; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; /** Safe startup classification produced before an application context is opened. */ -public record StartupDecision(RuntimeMode mode, SetupPhase phase, SetupErrorCode errorCode) { +public record StartupDecision(RuntimeMode mode) { public StartupDecision { Objects.requireNonNull(mode, "mode"); - Objects.requireNonNull(phase, "phase"); } public static StartupDecision normal() { - return new StartupDecision(RuntimeMode.NORMAL, SetupPhase.COMPLETE, null); + return new StartupDecision(RuntimeMode.NORMAL); } public static StartupDecision recovery() { - return new StartupDecision(RuntimeMode.RECOVERY, SetupPhase.RECOVERY_REQUIRED, - SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + return new StartupDecision(RuntimeMode.RECOVERY); } } diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupDecisionProbe.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupDecisionProbe.java index ce2e5b535c..9a8cf848b3 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupDecisionProbe.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupDecisionProbe.java @@ -21,5 +21,5 @@ package org.apache.hertzbeat.startup.runtime; @FunctionalInterface public interface StartupDecisionProbe { - StartupDecision probe(); + StartupDecision probe(String[] args); } diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbe.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbe.java index 74faf0e476..c6679bdb26 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbe.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbe.java @@ -19,7 +19,6 @@ package org.apache.hertzbeat.startup.runtime; import java.util.Objects; import org.apache.hertzbeat.common.runtime.RuntimeMode; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; /** Applies a local/container break-glass override before delegating to the installation probe. */ public final class StartupModePropertyProbe implements StartupDecisionProbe { @@ -30,7 +29,7 @@ public final class StartupModePropertyProbe implements StartupDecisionProbe { private final StartupDecisionProbe fallback; public StartupModePropertyProbe() { - this(StartupDecision::normal); + this(ignored -> StartupDecision.normal()); } public StartupModePropertyProbe(StartupDecisionProbe fallback) { @@ -38,26 +37,17 @@ public final class StartupModePropertyProbe implements StartupDecisionProbe { } @Override - public StartupDecision probe() { - return decide(System.getProperty(PROPERTY_NAME), System.getenv(ENVIRONMENT_NAME)); + public StartupDecision probe(String[] args) { + return decide(args, System.getProperty(PROPERTY_NAME), System.getenv(ENVIRONMENT_NAME)); } - StartupDecision decide(String systemValue, String environmentValue) { - String value = selectConfiguredValue(systemValue, environmentValue); - return value == null ? fallback.probe() : decisionFor(value); - } - - static String selectConfiguredValue(String systemValue, String environmentValue) { - return systemValue == null ? environmentValue : systemValue; + StartupDecision decide(String[] args, String systemValue, String environmentValue) { + String value = StartupArgumentProperties.resolve(args, PROPERTY_NAME, systemValue, environmentValue); + return value == null ? fallback.probe(args) : decisionFor(value); } static StartupDecision decisionFor(String value) { RuntimeMode mode = RuntimeMode.fromProperty(value); - return switch (mode) { - case NORMAL -> StartupDecision.normal(); - case SETUP_ONLY -> new StartupDecision(mode, SetupPhase.CONFIGURATION_REQUIRED, null); - case FULL_SETUP_GATED -> new StartupDecision(mode, SetupPhase.ADMINISTRATOR_REQUIRED, null); - case RECOVERY -> StartupDecision.recovery(); - }; + return new StartupDecision(mode); } } diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinatorTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinatorTest.java index 4b782e5bbb..2e90753aab 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinatorTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinatorTest.java @@ -22,21 +22,65 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import org.apache.hertzbeat.common.runtime.RuntimeMode; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.config.SetupInstallationPaths; import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; class HertzBeatStartupCoordinatorTest { + @TempDir + private Path installationRoot; + + @Test + void commandLineDatasourceParticipatesInThePreSpringDecision() { + String previous = System.getProperty(SetupInstallationPaths.ROOT_PROPERTY); + System.setProperty(SetupInstallationPaths.ROOT_PROPERTY, installationRoot.toString()); + try { + RecordingLauncher launcher = new RecordingLauncher(); + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( + new StartupModePropertyProbe(new LocalInstallationStartupProbe()), launcher); + + coordinator.start(new String[] { + "--" + SetupInstallationPaths.ROOT_PROPERTY + "=" + installationRoot, + "--spring.datasource.url=jdbc:postgresql://database.example.test/hertzbeat" + }); + + assertEquals(RuntimeMode.FULL_SETUP_GATED, coordinator.mode()); + } finally { + restoreSystemProperty(SetupInstallationPaths.ROOT_PROPERTY, previous); + } + } + + @Test + void commandLineStartupModeTakesPrecedenceOverSystemProperty() { + String previous = System.getProperty(StartupModePropertyProbe.PROPERTY_NAME); + System.setProperty(StartupModePropertyProbe.PROPERTY_NAME, RuntimeMode.FULL_SETUP_GATED.value()); + try { + RecordingLauncher launcher = new RecordingLauncher(); + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( + new StartupModePropertyProbe(ignored -> StartupDecision.normal()), launcher); + + coordinator.start(new String[] { + "--" + StartupModePropertyProbe.PROPERTY_NAME + "=" + RuntimeMode.SETUP_ONLY.value(), + "--" + RuntimeMode.PROPERTY_NAME + "=" + RuntimeMode.NORMAL.value() + }); + + assertEquals(RuntimeMode.SETUP_ONLY, coordinator.mode()); + } finally { + restoreSystemProperty(StartupModePropertyProbe.PROPERTY_NAME, previous); + } + } + @Test void startsFromProbeAndClosesGatedContextBeforeOpeningNormalExactlyOnce() { RecordingLauncher launcher = new RecordingLauncher(); - StartupDecision gated = new StartupDecision(RuntimeMode.FULL_SETUP_GATED, - SetupPhase.ADMINISTRATOR_REQUIRED, null); - HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator(() -> gated, launcher); + StartupDecision gated = new StartupDecision(RuntimeMode.FULL_SETUP_GATED); + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator(ignored -> gated, launcher); RunningApplicationContext first = coordinator.start(new String[]{"--server.port=0"}); SetupRuntimeTransition transition = launcher.transitions.getFirst(); @@ -57,11 +101,11 @@ class HertzBeatStartupCoordinatorTest { RecordingLauncher launcher = new RecordingLauncher(); launcher.failMode = RuntimeMode.FULL_SETUP_GATED; HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( - () -> new StartupDecision(RuntimeMode.SETUP_ONLY, SetupPhase.CONFIGURATION_REQUIRED, null), launcher); + ignored -> new StartupDecision(RuntimeMode.SETUP_ONLY), launcher); coordinator.start(new String[0]); - RunningApplicationContext recovery = coordinator.transition(new StartupDecision( - RuntimeMode.FULL_SETUP_GATED, SetupPhase.ADMINISTRATOR_REQUIRED, null)); + RunningApplicationContext recovery = coordinator.transition( + new StartupDecision(RuntimeMode.FULL_SETUP_GATED)); assertEquals(List.of("open:setup_only", "close:setup_only", "open:full_setup_gated", "open:recovery"), launcher.events); @@ -73,7 +117,7 @@ class HertzBeatStartupCoordinatorTest { void probeFailureCannotBeMisclassifiedAsNewInstallation() { RecordingLauncher launcher = new RecordingLauncher(); HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( - () -> { + ignored -> { throw new IllegalStateException("database unreachable"); }, launcher); @@ -88,7 +132,8 @@ class HertzBeatStartupCoordinatorTest { RecordingLauncher launcher = new RecordingLauncher(); launcher.failMode = RuntimeMode.NORMAL; launcher.failRecovery = true; - HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator(StartupDecision::normal, launcher); + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( + ignored -> StartupDecision.normal(), launcher); IllegalStateException failure = assertThrows(IllegalStateException.class, () -> coordinator.start(new String[0])); @@ -101,7 +146,8 @@ class HertzBeatStartupCoordinatorTest { @Test void nullContextIsAnExplicitLaunchFailure() { StartupContextLauncher launcher = (decision, args, transition) -> null; - HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator(StartupDecision::normal, launcher); + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( + ignored -> StartupDecision.normal(), launcher); NullPointerException failure = assertThrows(NullPointerException.class, () -> coordinator.start(new String[0])); @@ -111,6 +157,14 @@ class HertzBeatStartupCoordinatorTest { assertEquals("startup context launcher returned null for normal", failure.getSuppressed()[0].getMessage()); } + private static void restoreSystemProperty(String name, String value) { + if (value == null) { + System.clearProperty(name); + } else { + System.setProperty(name, value); + } + } + private static final class RecordingLauncher implements StartupContextLauncher { private final List events = new ArrayList<>(); diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbeTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbeTest.java index cd3061fce0..5195e0ff65 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbeTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbeTest.java @@ -31,6 +31,7 @@ import org.apache.hertzbeat.manager.setup.config.ManagedConfigurationTransaction import org.apache.hertzbeat.manager.setup.config.ManagedSecrets; import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.apache.hertzbeat.manager.setup.config.SetupInstallationPaths; import org.apache.hertzbeat.manager.setup.installation.LocalInstallationFingerprintStore; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -41,22 +42,45 @@ class LocalInstallationStartupProbeTest { @Test void freshRootStartsSetupOnlyAndLegacyDatabaseStartsGated() throws Exception { - assertEquals(RuntimeMode.SETUP_ONLY, new LocalInstallationStartupProbe(root, false).probe().mode()); + assertEquals(RuntimeMode.SETUP_ONLY, + new LocalInstallationStartupProbe(root, false).probe(new String[0]).mode()); Files.createDirectories(root.resolve("data")); Files.createFile(root.resolve("data/hertzbeat.mv.db")); assertEquals(RuntimeMode.FULL_SETUP_GATED, - new LocalInstallationStartupProbe(root, false).probe().mode()); + new LocalInstallationStartupProbe(root, false).probe(new String[0]).mode()); } @Test void localFingerprintCanNeverOpenBusinessRuntimeWithoutDatabaseComparison() throws Exception { new ManagedConfigurationTransaction(root).apply(bundle()); assertEquals(RuntimeMode.FULL_SETUP_GATED, - new LocalInstallationStartupProbe(root, false).probe().mode()); + new LocalInstallationStartupProbe(root, false).probe(new String[0]).mode()); new LocalInstallationFingerprintStore(root.resolve("data/config/.installation-fingerprint"), new SecureRandom()).create(); assertEquals(RuntimeMode.FULL_SETUP_GATED, - new LocalInstallationStartupProbe(root, false).probe().mode()); + new LocalInstallationStartupProbe(root, false).probe(new String[0]).mode()); + } + + @Test + void commandLineInstallationRootTakesPrecedenceOverSystemRoot() throws Exception { + Path systemRoot = Files.createDirectories(root.resolve("system-root/data")); + Files.createFile(systemRoot.resolve("hertzbeat.mv.db")); + Path commandLineRoot = Files.createDirectories(root.resolve("command-line-root")); + String previous = System.getProperty(SetupInstallationPaths.ROOT_PROPERTY); + System.setProperty(SetupInstallationPaths.ROOT_PROPERTY, systemRoot.getParent().toString()); + try { + StartupDecision decision = new LocalInstallationStartupProbe().probe(new String[] { + "--" + SetupInstallationPaths.ROOT_PROPERTY + "=" + commandLineRoot + }); + + assertEquals(RuntimeMode.SETUP_ONLY, decision.mode()); + } finally { + if (previous == null) { + System.clearProperty(SetupInstallationPaths.ROOT_PROPERTY); + } else { + System.setProperty(SetupInstallationPaths.ROOT_PROPERTY, previous); + } + } } private static ManagedConfigurationBundle bundle() { diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/ManagedConfigRecoveryContextTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/ManagedConfigRecoveryContextTest.java index 75a2b0cfe9..88de21e441 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/ManagedConfigRecoveryContextTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/ManagedConfigRecoveryContextTest.java @@ -23,7 +23,6 @@ import java.nio.file.Files; import java.nio.file.Path; import org.apache.hertzbeat.common.runtime.RuntimeMode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; import org.apache.hertzbeat.manager.setup.config.GreptimeEndpoints; import org.apache.hertzbeat.manager.setup.config.GreptimeSettings; import org.apache.hertzbeat.manager.setup.config.ManagedApplicationConfig; @@ -49,7 +48,7 @@ class ManagedConfigRecoveryContextTest { temporaryDirectory.resolve(brokenPair.name().toLowerCase(java.util.Locale.ROOT))); brokenPair.create(installationRoot); HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( - StartupDecision::normal, new SpringStartupContextLauncher()); + ignored -> StartupDecision.normal(), new SpringStartupContextLauncher()); coordinator.start(new String[] { "--hertzbeat.internal.installation-root=" + installationRoot, @@ -68,8 +67,7 @@ class ManagedConfigRecoveryContextTest { Path installationRoot = Files.createDirectories( temporaryDirectory.resolve("setup-" + brokenPair.name().toLowerCase(java.util.Locale.ROOT))); brokenPair.create(installationRoot); - StartupDecision setupOnly = new StartupDecision( - RuntimeMode.SETUP_ONLY, SetupPhase.CONFIGURATION_REQUIRED, null); + StartupDecision setupOnly = new StartupDecision(RuntimeMode.SETUP_ONLY); try (ConfigurableApplicationContext context = new SpringStartupContextLauncher().launchSpringContext( setupOnly, diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupArgumentPropertiesTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupArgumentPropertiesTest.java new file mode 100644 index 0000000000..b7486fa3d0 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupArgumentPropertiesTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup.runtime; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.nio.file.Path; +import java.util.Arrays; +import java.util.stream.Stream; +import org.apache.hertzbeat.manager.setup.config.SetupInstallationPaths; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.context.annotation.Configuration; + +class StartupArgumentPropertiesTest { + @TempDir + private Path installationRoot; + + @ParameterizedTest(name = "{0}") + @MethodSource("commandLineForms") + void preSpringResolutionMatchesLaunchedEnvironment(String ignored, String[] commandLine) { + String resolved = StartupArgumentProperties.resolve(commandLine, + StartupModePropertyProbe.PROPERTY_NAME, "system-value", "environment-value"); + String[] launchArguments = Arrays.copyOf(commandLine, commandLine.length + 1); + launchArguments[commandLine.length] = "--" + SetupInstallationPaths.ROOT_PROPERTY + + "=" + installationRoot; + + try (var context = new SpringApplicationBuilder(ArgumentApplication.class) + .web(WebApplicationType.NONE) + .logStartupInfo(false) + .properties("spring.main.banner-mode=off") + .run(launchArguments)) { + assertEquals(context.getEnvironment().getProperty(StartupModePropertyProbe.PROPERTY_NAME), resolved); + } + } + + private static Stream commandLineForms() { + String option = "--" + StartupModePropertyProbe.PROPERTY_NAME; + return Stream.of( + Arguments.of("bare option", (Object) new String[] {option}), + Arguments.of("value option", (Object) new String[] {option + "=setup_only"}), + Arguments.of("duplicate option", (Object) new String[] { + option + "=setup_only", option + "=normal" + })); + } + + @Configuration(proxyBeanMethods = false) + static class ArgumentApplication { + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbeTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbeTest.java index 670ea1e875..eb4d4cc642 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbeTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbeTest.java @@ -21,47 +21,45 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import org.apache.hertzbeat.common.runtime.RuntimeMode; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; import org.junit.jupiter.api.Test; class StartupModePropertyProbeTest { @Test void missingOverridePreservesNormalStartup() { - StartupDecision decision = new StartupModePropertyProbe().decide(null, null); + StartupDecision decision = new StartupModePropertyProbe().decide(new String[0], null, null); assertEquals(RuntimeMode.NORMAL, decision.mode()); - assertEquals(SetupPhase.COMPLETE, decision.phase()); } @Test void systemPropertyTakesPrecedenceOverEnvironment() { - StartupDecision decision = new StartupModePropertyProbe().decide("full_setup_gated", "setup_only"); + StartupDecision decision = new StartupModePropertyProbe().decide( + new String[0], "full_setup_gated", "setup_only"); assertEquals(RuntimeMode.FULL_SETUP_GATED, decision.mode()); - assertEquals(SetupPhase.ADMINISTRATOR_REQUIRED, decision.phase()); } @Test void environmentSelectsSetupOnlyWhenSystemPropertyIsMissing() { - StartupDecision decision = new StartupModePropertyProbe().decide(null, "setup_only"); + StartupDecision decision = new StartupModePropertyProbe().decide(new String[0], null, "setup_only"); assertEquals(RuntimeMode.SETUP_ONLY, decision.mode()); - assertEquals(SetupPhase.CONFIGURATION_REQUIRED, decision.phase()); } @Test void invalidOverrideFailsClosedForCoordinatorRecovery() { StartupModePropertyProbe probe = new StartupModePropertyProbe(); - assertThrows(IllegalArgumentException.class, () -> probe.decide("unsupported", null)); + assertThrows(IllegalArgumentException.class, + () -> probe.decide(new String[0], "unsupported", null)); } @Test void missingOverrideDelegatesToInstallationProbe() { StartupDecision expected = StartupDecision.recovery(); - StartupDecisionProbe fallback = () -> expected; + StartupDecisionProbe fallback = ignored -> expected; - assertEquals(expected, new StartupModePropertyProbe(fallback).decide(null, null)); + assertEquals(expected, new StartupModePropertyProbe(fallback).decide(new String[0], null, null)); } } diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupRuntimeBoundaryContextTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupRuntimeBoundaryContextTest.java index 8795bdb464..50dd1be331 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupRuntimeBoundaryContextTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupRuntimeBoundaryContextTest.java @@ -19,6 +19,8 @@ package org.apache.hertzbeat.startup.runtime; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -27,16 +29,21 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.file.Path; +import java.time.Clock; import org.apache.hertzbeat.common.runtime.BusinessRuntimeGate; import org.apache.hertzbeat.common.runtime.RuntimeMode; import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; +import org.apache.hertzbeat.manager.setup.workflow.SetupOperationRegistry; import org.apache.hertzbeat.manager.setup.workflow.SetupRuntimeState; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.springframework.boot.web.server.context.WebServerApplicationContext; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; class StartupRuntimeBoundaryContextTest { @@ -59,11 +66,41 @@ class StartupRuntimeBoundaryContextTest { @TempDir Path installationRoot; + @Test + void awaitingRestartOperationConvergesAcrossContextRebuild() { + String operationId = createAwaitingRestartOperation(installationRoot.resolve("success")); + try (ConfigurableApplicationContext context = operationContext( + installationRoot.resolve("success"), SetupPhase.ADMINISTRATOR_REQUIRED)) { + var operation = context.getBean(SetupOperationRegistry.class).get(operationId); + + assertNotNull(operation); + assertEquals(SetupOperationState.SUCCEEDED, operation.state()); + assertEquals(SetupPhase.ADMINISTRATOR_REQUIRED, operation.phase()); + assertNotNull(operation.completedAt()); + assertEquals(0, operation.nextPollAfterMillis()); + assertNull(context.getBean(SetupOperationRegistry.class).get("unknown-operation")); + } + } + + @Test + void awaitingRestartOperationBecomesFailedDuringRecovery() { + Path root = installationRoot.resolve("recovery"); + String operationId = createAwaitingRestartOperation(root); + try (ConfigurableApplicationContext context = operationContext(root, SetupPhase.RECOVERY_REQUIRED)) { + var operation = context.getBean(SetupOperationRegistry.class).get(operationId); + + assertNotNull(operation); + assertEquals(SetupOperationState.FAILED, operation.state()); + assertEquals(SetupPhase.RECOVERY_REQUIRED, operation.phase()); + assertEquals(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, operation.errorCode()); + assertNotNull(operation.completedAt()); + } + } + @Test void fullGatedSurenessChainKeepsSetupReachableAndBusinessRoutesClosed() throws Exception { SpringStartupContextLauncher launcher = new SpringStartupContextLauncher(); - StartupDecision decision = new StartupDecision( - RuntimeMode.FULL_SETUP_GATED, SetupPhase.ADMINISTRATOR_REQUIRED, null); + StartupDecision decision = new StartupDecision(RuntimeMode.FULL_SETUP_GATED); String databaseName = "m5_setup_security_" + System.nanoTime(); try (ConfigurableApplicationContext context = launcher.launchSpringContext(decision, new String[]{ "--spring.profiles.active=test", @@ -105,8 +142,7 @@ class StartupRuntimeBoundaryContextTest { @Test void realFullApplicationStartsGatedWithoutBusinessSideEffectsOrCliBypass() { SpringStartupContextLauncher launcher = new SpringStartupContextLauncher(); - StartupDecision decision = new StartupDecision( - RuntimeMode.FULL_SETUP_GATED, SetupPhase.ADMINISTRATOR_REQUIRED, null); + StartupDecision decision = new StartupDecision(RuntimeMode.FULL_SETUP_GATED); String databaseName = "m2_gated_" + System.nanoTime(); try (ConfigurableApplicationContext context = launcher.launchSpringContext(decision, new String[]{ "--spring.profiles.active=test", @@ -148,8 +184,7 @@ class StartupRuntimeBoundaryContextTest { @Test void setupOnlySourceStartsWithoutBusinessAutoConfiguration() { SpringStartupContextLauncher launcher = new SpringStartupContextLauncher(); - StartupDecision decision = new StartupDecision( - RuntimeMode.SETUP_ONLY, SetupPhase.CONFIGURATION_REQUIRED, null); + StartupDecision decision = new StartupDecision(RuntimeMode.SETUP_ONLY); try (ConfigurableApplicationContext context = launcher.launchSpringContext(decision, new String[]{"--spring.main.web-application-type=none", "--hertzbeat.runtime.mode=normal"}, SETUP_RUNTIME_TRANSITION)) { @@ -171,4 +206,22 @@ class StartupRuntimeBoundaryContextTest { private static HttpRequest.Builder request(int port, String path) { return HttpRequest.newBuilder(URI.create("http://127.0.0.1:" + port + path)); } + + private static String createAwaitingRestartOperation(Path root) { + try (ConfigurableApplicationContext context = operationContext(root, SetupPhase.CONFIGURATION_REQUIRED)) { + SetupOperationRegistry operations = context.getBean(SetupOperationRegistry.class); + String operationId = operations.begin(SetupPhase.CONFIGURATION_REQUIRED); + operations.finish(operationId, SetupOperationState.AWAITING_RESTART, + SetupPhase.APPLICATION_STARTING, null, false); + return operationId; + } + } + + private static ConfigurableApplicationContext operationContext(Path root, SetupPhase phase) { + AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); + context.registerBean(SetupOperationRegistry.class, + () -> new SetupOperationRegistry(Clock.systemUTC(), root, phase)); + context.refresh(); + return context; + } } From 697c36e20c9012563c101eb88bd6962f011d0130 Mon Sep 17 00:00:00 2001 From: Logic Date: Sun, 9 Aug 2026 02:48:12 +0800 Subject: [PATCH 17/71] Preserve safe setup failure diagnostics --- .../setup/api/SetupExceptionHandler.java | 14 ++- .../LoggingRecoveryFailureReporter.java | 14 ++- .../setup/config/RecoveryFailureReporter.java | 12 +- .../api/SetupExceptionHandlerLoggingTest.java | 112 ++++++++++++++++++ .../LoggingRecoveryFailureReporterTest.java | 69 +++++++++++ .../ManagedConfigurationTransactionTest.java | 30 +++-- 6 files changed, 219 insertions(+), 32 deletions(-) create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandlerLoggingTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/LoggingRecoveryFailureReporterTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandler.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandler.java index d14798b3b9..56adc5cfa8 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandler.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandler.java @@ -21,6 +21,8 @@ import java.time.Clock; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; import org.apache.hertzbeat.manager.setup.security.SetupUnlockRejected; import org.apache.hertzbeat.manager.setup.workflow.SetupWorkflowConflict; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.http.converter.HttpMessageNotReadableException; @@ -31,6 +33,8 @@ import org.springframework.web.bind.annotation.RestControllerAdvice; /** Owns safe HTTP classification for setup failures. */ @RestControllerAdvice(assignableTypes = SetupController.class) public class SetupExceptionHandler { + private static final Logger LOGGER = LoggerFactory.getLogger(SetupExceptionHandler.class); + private final Clock clock; public SetupExceptionHandler() { @@ -66,10 +70,18 @@ public class SetupExceptionHandler { } @ExceptionHandler(Exception.class) - public ResponseEntity unexpectedFailure(Exception ignored) { + public ResponseEntity unexpectedFailure(Exception failure) { + LOGGER.error("Unexpected setup request failure exception={}", + failure.getClass().getName(), diagnosticCopy(failure)); return response(HttpStatus.INTERNAL_SERVER_ERROR, SetupErrorCode.INTERNAL_ERROR); } + private static Throwable diagnosticCopy(Throwable failure) { + Throwable diagnostic = new Throwable(); + diagnostic.setStackTrace(failure.getStackTrace()); + return diagnostic; + } + private ResponseEntity response(HttpStatus status, SetupErrorCode code) { return ResponseEntity.status(status).header("Cache-Control", "no-store") .body(new SetupErrorResponse(code, clock.instant())); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/LoggingRecoveryFailureReporter.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/LoggingRecoveryFailureReporter.java index f039c2f6ff..73f0421dda 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/LoggingRecoveryFailureReporter.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/LoggingRecoveryFailureReporter.java @@ -10,13 +10,19 @@ package org.apache.hertzbeat.manager.setup.config; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** Production adapter that logs only the pre-sanitized recovery diagnostic event. */ +/** Production adapter that logs safe recovery diagnostics. */ final class LoggingRecoveryFailureReporter implements RecoveryFailureReporter { private static final Logger LOGGER = LoggerFactory.getLogger(LoggingRecoveryFailureReporter.class); @Override - public void report(Failure failure) { - LOGGER.warn("Managed configuration recovery failure stage={} store={} exception={} detail={}", - failure.stage(), failure.store(), failure.exceptionClass(), failure.safeMessage()); + public void report(Stage stage, Store store, Exception failure) { + LOGGER.warn("Managed configuration recovery failure stage={} store={} exception={}", + stage, store, failure.getClass().getName(), diagnosticCopy(failure)); + } + + private static Throwable diagnosticCopy(Throwable failure) { + Throwable diagnostic = new Throwable(); + diagnostic.setStackTrace(failure.getStackTrace()); + return diagnostic; } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/RecoveryFailureReporter.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/RecoveryFailureReporter.java index f7c0941df1..a926a6ce1b 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/RecoveryFailureReporter.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/RecoveryFailureReporter.java @@ -10,13 +10,7 @@ package org.apache.hertzbeat.manager.setup.config; /** Secret-free diagnostic boundary for managed configuration recovery failures. */ @FunctionalInterface public interface RecoveryFailureReporter { - String SAFE_MESSAGE = "Managed configuration recovery operation failed"; - - void report(Failure failure); - - default void report(Stage stage, Store store, Exception failure) { - report(new Failure(stage, store, failure.getClass().getName(), SAFE_MESSAGE)); - } + void report(Stage stage, Store store, Exception failure); /** Recovery operation stage that failed. */ enum Stage { @@ -30,8 +24,4 @@ public interface RecoveryFailureReporter { APPLICATION, SECRET } - - /** Fully sanitized diagnostic event safe for production logging. */ - record Failure(Stage stage, Store store, String exceptionClass, String safeMessage) { - } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandlerLoggingTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandlerLoggingTest.java new file mode 100644 index 0000000000..570ebacb8a --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandlerLoggingTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.runtime.SetupResponseTransition; +import org.apache.hertzbeat.manager.setup.security.SetupHttpUnlockService; +import org.apache.hertzbeat.manager.setup.workflow.SetupExportRenderer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; + +class SetupExceptionHandlerLoggingTest { + private final SetupWorkflow workflow = mock(SetupWorkflow.class); + private final Logger logger = (Logger) LoggerFactory.getLogger(SetupExceptionHandler.class); + private final ListAppender appender = new ListAppender<>(); + private MockMvc mvc; + + @BeforeEach + void setUp() { + appender.start(); + logger.addAppender(appender); + mvc = MockMvcBuilders.standaloneSetup(new SetupController(workflow, + mock(SetupHttpUnlockService.class), mock(SetupResponseTransition.class), + new SetupExportRenderer())) + .setControllerAdvice(new SetupExceptionHandler()).build(); + } + + @AfterEach + void tearDown() { + logger.detachAppender(appender); + appender.stop(); + } + + @Test + void unexpectedFailureLogsFixedContextAndThrowableWhileResponseStaysSafe() throws Exception { + IllegalStateException failure = new IllegalStateException("exception-secret"); + when(workflow.status()).thenThrow(failure); + + mvc.perform(get(SetupApiContract.STATUS_PATH).queryParam("token", "query-secret")) + .andExpect(status().isInternalServerError()) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.errorCode").value("internal_error")) + .andExpect(content().string(org.hamcrest.Matchers.not( + org.hamcrest.Matchers.containsString("exception-secret")))) + .andExpect(content().string(org.hamcrest.Matchers.not( + org.hamcrest.Matchers.containsString("query-secret")))); + + var errors = appender.list.stream().filter(event -> event.getLevel() == Level.ERROR).toList(); + assertEquals(1, errors.size()); + ILoggingEvent event = errors.getFirst(); + assertEquals("Unexpected setup request failure exception=java.lang.IllegalStateException", + event.getFormattedMessage()); + assertNotNull(event.getThrowableProxy()); + assertEquals(Throwable.class.getName(), event.getThrowableProxy().getClassName()); + assertNull(event.getThrowableProxy().getMessage()); + assertNull(event.getThrowableProxy().getCause()); + assertTrue(event.getThrowableProxy().getStackTraceElementProxyArray().length > 0); + assertEquals(failure.getStackTrace()[0], + event.getThrowableProxy().getStackTraceElementProxyArray()[0].getStackTraceElement()); + } + + @Test + void typedAndInvalidRequestsAreNotLoggedAsUnexpected() throws Exception { + when(workflow.status()).thenThrow( + new SetupApiException(SetupErrorCode.CONFIG_READ_ONLY, HttpStatus.CONFLICT)); + + mvc.perform(get(SetupApiContract.STATUS_PATH)).andExpect(status().isConflict()); + mvc.perform(post(SetupApiContract.UNLOCK_PATH) + .contentType(MediaType.APPLICATION_JSON).content("{}")) + .andExpect(status().isBadRequest()); + + assertTrue(appender.list.stream().noneMatch(event -> event.getLevel() == Level.ERROR)); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/LoggingRecoveryFailureReporterTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/LoggingRecoveryFailureReporterTest.java new file mode 100644 index 0000000000..e5352b58c0 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/LoggingRecoveryFailureReporterTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import java.io.IOException; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +class LoggingRecoveryFailureReporterTest { + + @Test + void logsFixedStageStoreContextAndSafeDiagnosticThrowable() { + Logger logger = (Logger) LoggerFactory.getLogger(LoggingRecoveryFailureReporter.class); + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + try { + IOException failure = new IOException("password=secret jdbc:postgresql://private/path"); + new LoggingRecoveryFailureReporter().report( + RecoveryFailureReporter.Stage.PROMOTE_CANDIDATE, + RecoveryFailureReporter.Store.SECRET, + failure); + + assertEquals(1, appender.list.size()); + ILoggingEvent event = appender.list.getFirst(); + assertEquals(Level.WARN, event.getLevel()); + assertEquals("Managed configuration recovery failure stage=PROMOTE_CANDIDATE store=SECRET " + + "exception=java.io.IOException", + event.getFormattedMessage()); + assertFalse(event.getFormattedMessage().contains("password")); + assertFalse(event.getFormattedMessage().contains("jdbc")); + assertNotNull(event.getThrowableProxy()); + assertEquals(Throwable.class.getName(), event.getThrowableProxy().getClassName()); + assertNull(event.getThrowableProxy().getMessage()); + assertNull(event.getThrowableProxy().getCause()); + assertTrue(event.getThrowableProxy().getStackTraceElementProxyArray().length > 0); + assertEquals(failure.getStackTrace()[0], + event.getThrowableProxy().getStackTraceElementProxyArray()[0].getStackTraceElement()); + } finally { + logger.detachAppender(appender); + appender.stop(); + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java index 3540ce9b82..054ef0af4b 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java @@ -21,7 +21,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.eq; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.same; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.IOException; @@ -31,8 +35,6 @@ import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.util.ArrayList; -import java.util.List; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; @@ -64,17 +66,15 @@ class ManagedConfigurationTransactionTest { FileManagedApplicationConfigStore applicationStore = new FileManagedApplicationConfigStore(installationRoot); FileManagedSecretStore failingSecrets = new FileManagedSecretStore( installationRoot, new FailingOnceActivePublicationPublisher(new NioManagedFilePublisher())); - List diagnostics = new ArrayList<>(); + RecoveryFailureReporter diagnostics = mock(RecoveryFailureReporter.class); ManagedConfigurationTransaction transaction = new ManagedConfigurationTransaction( - applicationStore, failingSecrets, installationRoot, diagnostics::add); + applicationStore, failingSecrets, installationRoot, diagnostics); assertEquals(ManagedConfigurationTransaction.Outcome.ROLLED_BACK, transaction.apply(bundle("next"))); assertActivePair("previous"); - assertThat(diagnostics).containsExactly(new RecoveryFailureReporter.Failure( - RecoveryFailureReporter.Stage.PROMOTE_CANDIDATE, - RecoveryFailureReporter.Store.SECRET, IOException.class.getName(), - RecoveryFailureReporter.SAFE_MESSAGE)); + verify(diagnostics).report(eq(RecoveryFailureReporter.Stage.PROMOTE_CANDIDATE), + eq(RecoveryFailureReporter.Store.SECRET), any(IOException.class)); } @Test @@ -206,18 +206,16 @@ class ManagedConfigurationTransactionTest { when(secretStore.readActive()).thenReturn(CandidateRead.valid(secrets("owned"), "generation")); when(secretStore.readCandidate()).thenReturn(CandidateRead.missing()); when(secretStore.readLastKnownGood()).thenReturn(CandidateRead.missing()); - doThrow(new IOException("must-not-be-reported")).when(applications).discardCandidate(); - List diagnostics = new ArrayList<>(); + IOException failure = new IOException("must-not-be-reported"); + doThrow(failure).when(applications).discardCandidate(); + RecoveryFailureReporter diagnostics = mock(RecoveryFailureReporter.class); assertEquals(ManagedConfigurationTransaction.Outcome.RECOVERY_REQUIRED, new ManagedConfigurationTransaction( - applications, secretStore, installationRoot, diagnostics::add).recover()); + applications, secretStore, installationRoot, diagnostics).recover()); - assertThat(diagnostics).containsExactly(new RecoveryFailureReporter.Failure( - RecoveryFailureReporter.Stage.DISCARD_CANDIDATE, - RecoveryFailureReporter.Store.APPLICATION, IOException.class.getName(), - RecoveryFailureReporter.SAFE_MESSAGE)); - assertThat(diagnostics.getFirst().toString()).doesNotContain("must-not-be-reported"); + verify(diagnostics).report(eq(RecoveryFailureReporter.Stage.DISCARD_CANDIDATE), + eq(RecoveryFailureReporter.Store.APPLICATION), same(failure)); } private static void waitForFile(Path ready) throws Exception { From 6e63b80e7253d05e7c4ce6a3c7534126138afb25 Mon Sep 17 00:00:00 2001 From: Logic Date: Sun, 9 Aug 2026 03:51:04 +0800 Subject: [PATCH 18/71] Harden setup JDBC probe lifecycle --- .../workflow/JdbcMetadataConnectionProbe.java | 157 +--------- .../workflow/JdbcMetadataProbeCleanup.java | 103 +++++++ .../JdbcMetadataProbeFailureClassifier.java | 51 ++++ .../workflow/JdbcMetadataProbeOperations.java | 153 ++++++++++ .../workflow/JdbcMetadataProbeSession.java | 224 ++++++++++++++ ...dbcMetadataConnectionProbeCleanupTest.java | 229 ++++++++++++++ ...etadataConnectionProbeConcurrencyTest.java | 282 ++++++++++++++++++ .../JdbcMetadataConnectionProbeTest.java | 70 ++++- .../JdbcMetadataProbeCleanupBoundaryTest.java | 132 ++++++++ 9 files changed, 1244 insertions(+), 157 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataProbeCleanup.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataProbeFailureClassifier.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataProbeOperations.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataProbeSession.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataConnectionProbeCleanupTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataConnectionProbeConcurrencyTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataProbeCleanupBoundaryTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataConnectionProbe.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataConnectionProbe.java index b15fccb2cc..a5517a41d9 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataConnectionProbe.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataConnectionProbe.java @@ -18,25 +18,14 @@ package org.apache.hertzbeat.manager.setup.workflow; import java.sql.Connection; -import java.sql.DatabaseMetaData; import java.sql.DriverManager; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; import java.time.Duration; -import java.util.Arrays; -import java.util.Locale; import java.util.Optional; -import java.util.UUID; import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.Executor; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicReference; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; /** Bounded JDBC connection, dialect/charset, schema access, and temporary DDL/DML probe. */ @@ -52,154 +41,30 @@ public final class JdbcMetadataConnectionProbe implements MetadataConnectionProb }, new ThreadPoolExecutor.AbortPolicy()); private final Duration timeout; private final ThreadPoolExecutor executor; + private final Executor cleanupExecutor; private final JdbcConnector connector; public JdbcMetadataConnectionProbe(Duration timeout) { - this(timeout, SHARED_EXECUTOR, (url, username, password) -> + this(timeout, SHARED_EXECUTOR, JdbcMetadataProbeCleanup.sharedExecutor(), (url, username, password) -> DriverManager.getConnection(url, username, new String(password))); } JdbcMetadataConnectionProbe(Duration timeout, ThreadPoolExecutor executor, JdbcConnector connector) { + this(timeout, executor, Runnable::run, connector); + } + + JdbcMetadataConnectionProbe(Duration timeout, ThreadPoolExecutor executor, + Executor cleanupExecutor, JdbcConnector connector) { this.timeout = timeout; this.executor = executor; + this.cleanupExecutor = cleanupExecutor; this.connector = connector; } @Override public Optional probe(MetadataConnectionProbe.Request configuration) { - AtomicReference activeConnection = new AtomicReference<>(); - Future> future; - try { - future = executor.submit(() -> validate(configuration, activeConnection)); - } catch (RejectedExecutionException overload) { - return Optional.of(SetupErrorCode.METADATA_CONNECTION_FAILED); - } - try { - return future.get(timeout.toMillis(), TimeUnit.MILLISECONDS); - } catch (InterruptedException failure) { - Thread.currentThread().interrupt(); - return Optional.of(SetupErrorCode.METADATA_CONNECTION_FAILED); - } catch (ExecutionException | TimeoutException failure) { - future.cancel(true); - close(activeConnection.get()); - return Optional.of(SetupErrorCode.METADATA_CONNECTION_FAILED); - } - } - - private Optional validate(MetadataConnectionProbe.Request configuration, - AtomicReference activeConnection) { - char[] password = configuration.password().copy(); - try (Connection connection = connector.connect(configuration.jdbcUrl(), - configuration.username(), password)) { - activeConnection.set(connection); - connection.setNetworkTimeout(Runnable::run, Math.toIntExact(timeout.toMillis())); - Optional compatibility = validateCompatibility(connection, configuration.kind()); - if (compatibility.isPresent()) { - return compatibility; - } - return validatePrivileges(connection); - } catch (SQLException | RuntimeException failure) { - return Optional.of(SetupErrorCode.METADATA_CONNECTION_FAILED); - } finally { - Arrays.fill(password, '\0'); - activeConnection.set(null); - } - } - - private Optional validateCompatibility(Connection connection, - MetadataDatabaseKind expected) throws SQLException { - DatabaseMetaData metadata = connection.getMetaData(); - String product = metadata.getDatabaseProductName().toLowerCase(Locale.ROOT); - boolean matches = switch (expected) { - case H2 -> product.contains("h2"); - case MYSQL -> product.contains("mysql"); - case POSTGRESQL -> product.contains("postgresql"); - }; - if (!matches || !utf8Compatible(connection, expected)) { - return Optional.of(SetupErrorCode.METADATA_SCHEMA_MISMATCH); - } - try (ResultSet ignored = metadata.getSchemas()) { - // Opening the schema projection verifies that metadata visibility is available. - } - return Optional.empty(); - } - - private boolean utf8Compatible(Connection connection, MetadataDatabaseKind kind) throws SQLException { - String sql = switch (kind) { - case MYSQL -> "SELECT @@character_set_database"; - case POSTGRESQL -> "SHOW server_encoding"; - case H2 -> null; - }; - if (sql == null) { - return true; - } - try (Statement statement = connection.createStatement()) { - statement.setQueryTimeout(Math.max(1, Math.toIntExact(timeout.toSeconds()))); - try (ResultSet result = statement.executeQuery(sql)) { - return result.next() && result.getString(1).toLowerCase(Locale.ROOT).startsWith("utf8"); - } - } - } - - Optional validatePrivileges(Connection connection) { - String table = "HZB_SETUP_PROBE_" + UUID.randomUUID().toString().replace("-", ""); - boolean created = false; - try { - connection.setAutoCommit(false); - try (Statement statement = connection.createStatement()) { - statement.setQueryTimeout(Math.max(1, Math.toIntExact(timeout.toSeconds()))); - statement.execute("CREATE TABLE " + table - + " (probe_id INTEGER NOT NULL PRIMARY KEY, probe_value VARCHAR(32) NOT NULL)"); - created = true; - statement.executeUpdate("INSERT INTO " + table + " VALUES (1, 'created')"); - statement.executeUpdate("UPDATE " + table + " SET probe_value = 'updated' WHERE probe_id = 1"); - try (ResultSet result = statement.executeQuery("SELECT probe_value FROM " + table - + " WHERE probe_id = 1")) { - if (!result.next() || !"updated".equals(result.getString(1))) { - throw new SQLException("Temporary probe value was not readable"); - } - } - statement.executeUpdate("DELETE FROM " + table + " WHERE probe_id = 1"); - statement.execute("DROP TABLE " + table); - created = false; - } - connection.commit(); - return Optional.empty(); - } catch (SQLException failure) { - rollback(connection); - if (created) { - drop(connection, table); - } - return Optional.of(SetupErrorCode.METADATA_INSUFFICIENT_PRIVILEGES); - } - } - - private static boolean drop(Connection connection, String table) { - try (Statement statement = connection.createStatement()) { - statement.execute("DROP TABLE " + table); - connection.commit(); - return true; - } catch (SQLException ignored) { - return false; - } - } - - private static void rollback(Connection connection) { - try { - connection.rollback(); - } catch (SQLException ignored) { - // Preserve the stable validation error. - } - } - - private static void close(Connection connection) { - if (connection != null) { - try { - connection.close(); - } catch (SQLException ignored) { - // Preserve the stable timeout error. - } - } + return new JdbcMetadataProbeSession( + configuration, timeout, executor, cleanupExecutor, connector).probe(); } @FunctionalInterface diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataProbeCleanup.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataProbeCleanup.java new file mode 100644 index 0000000000..c39515ce69 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataProbeCleanup.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.concurrent.Executor; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Runs exact-candidate cleanup outside the capacity-limited primary probe executor. */ +final class JdbcMetadataProbeCleanup { + private static final Logger LOGGER = LoggerFactory.getLogger(JdbcMetadataConnectionProbe.class); + private static final int MAX_CONCURRENT_CLEANUPS = 1; + private static final ThreadPoolExecutor CLEANUP_EXECUTOR = new ThreadPoolExecutor( + MAX_CONCURRENT_CLEANUPS, MAX_CONCURRENT_CLEANUPS, 0L, TimeUnit.MILLISECONDS, + new SynchronousQueue<>(), task -> { + Thread thread = new Thread(task, "setup-metadata-probe-cleanup"); + thread.setDaemon(true); + return thread; + }, new ThreadPoolExecutor.AbortPolicy()); + private final MetadataConnectionProbe.Request request; + private final long timeoutNanos; + private final Executor cleanupExecutor; + private final JdbcMetadataConnectionProbe.JdbcConnector connector; + private final JdbcMetadataProbeOperations operations = new JdbcMetadataProbeOperations(); + + JdbcMetadataProbeCleanup(MetadataConnectionProbe.Request request, long timeoutNanos, + Executor cleanupExecutor, JdbcMetadataConnectionProbe.JdbcConnector connector) { + this.request = request; + this.timeoutNanos = timeoutNanos; + this.cleanupExecutor = cleanupExecutor; + this.connector = connector; + } + + void schedule(String table, char[] sourcePassword, Runnable confirmed) { + char[] password = Arrays.copyOf(sourcePassword, sourcePassword.length); + try { + // Zero queue capacity prevents cleartext credential copies from waiting behind a stuck connector. + cleanupExecutor.execute(() -> cleanup(table, password, confirmed)); + } catch (RejectedExecutionException overload) { + Arrays.fill(password, '\0'); + logFailure(table, null, 0); + } + } + + static Executor sharedExecutor() { + return CLEANUP_EXECUTOR; + } + + private void cleanup(String table, char[] password, Runnable confirmed) { + Thread.interrupted(); + long deadline = JdbcMetadataProbeOperations.deadlineAfter(timeoutNanos); + // JDBC connect has no portable hard timeout; isolation keeps this best-effort wait out of the probe pool. + try (Connection connection = connector.connect(request.jdbcUrl(), request.username(), password)) { + operations.configureConnection(connection, deadline); + connection.setAutoCommit(false); + operations.dropIfExists(connection, table, () -> deadline); + connection.commit(); + confirmed.run(); + } catch (SQLException failure) { + logFailure(table, failure.getSQLState(), failure.getErrorCode()); + } catch (RuntimeException failure) { + logFailure(table, null, 0); + } finally { + Arrays.fill(password, '\0'); + } + } + + private void logFailure(String table, String sqlState, int vendorCode) { + LOGGER.warn("Metadata probe cleanup failure kind={} table={} sqlState={} vendorCode={}", + request.kind(), table, safeSqlState(sqlState), vendorCode); + } + + private static String safeSqlState(String sqlState) { + if (sqlState == null || sqlState.length() != 5) { + return "unknown"; + } + return sqlState.chars().allMatch(character -> character >= '0' && character <= '9' + || character >= 'A' && character <= 'Z') + ? sqlState : "unknown"; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataProbeFailureClassifier.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataProbeFailureClassifier.java new file mode 100644 index 0000000000..cca870ce8f --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataProbeFailureClassifier.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.SQLException; +import java.sql.SQLNonTransientConnectionException; +import java.sql.SQLRecoverableException; +import java.sql.SQLTimeoutException; +import java.sql.SQLTransientConnectionException; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Separates connectivity loss from genuine DDL/DML privilege rejection. */ +final class JdbcMetadataProbeFailureClassifier { + + private JdbcMetadataProbeFailureClassifier() { + } + + static SetupErrorCode classify(SQLException failure) { + String sqlState = failure.getSQLState(); + if (failure instanceof SQLTransientConnectionException + || failure instanceof SQLNonTransientConnectionException + || failure instanceof SQLRecoverableException + || failure instanceof SQLTimeoutException + || isConnectionOrTimeoutState(sqlState)) { + return SetupErrorCode.METADATA_CONNECTION_FAILED; + } + return SetupErrorCode.METADATA_INSUFFICIENT_PRIVILEGES; + } + + private static boolean isConnectionOrTimeoutState(String sqlState) { + return sqlState != null && (sqlState.startsWith("08") + || "HYT00".equals(sqlState) + || "HYT01".equals(sqlState) + || "57014".equals(sqlState)); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataProbeOperations.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataProbeOperations.java new file mode 100644 index 0000000000..f98b0bdd82 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataProbeOperations.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Locale; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Executes the compatibility and temporary CRUD statements for one session-owned candidate. */ +final class JdbcMetadataProbeOperations { + + Optional validateCompatibility(Connection connection, MetadataDatabaseKind expected, + DeadlineGuard deadline) throws SQLException { + deadline.activeDeadline(); + DatabaseMetaData metadata = connection.getMetaData(); + String product = metadata.getDatabaseProductName().toLowerCase(Locale.ROOT); + boolean matches = switch (expected) { + case H2 -> product.contains("h2"); + case MYSQL -> product.contains("mysql"); + case POSTGRESQL -> product.contains("postgresql"); + }; + if (!matches || !utf8Compatible(connection, expected, deadline)) { + return Optional.of(SetupErrorCode.METADATA_SCHEMA_MISMATCH); + } + deadline.activeDeadline(); + try (ResultSet ignored = metadata.getSchemas()) { + // Opening the schema projection verifies that metadata visibility is available. + } + return Optional.empty(); + } + + void executeCrudProbe(Connection connection, String table, DeadlineGuard deadline) throws SQLException { + try (Statement statement = connection.createStatement()) { + execute(statement, "CREATE TABLE " + table + + " (probe_id INTEGER NOT NULL PRIMARY KEY, probe_value VARCHAR(32) NOT NULL)", deadline); + executeUpdate(statement, "INSERT INTO " + table + " VALUES (1, 'created')", deadline); + executeUpdate(statement, + "UPDATE " + table + " SET probe_value = 'updated' WHERE probe_id = 1", deadline); + prepare(statement, deadline.activeDeadline()); + try (ResultSet result = statement.executeQuery("SELECT probe_value FROM " + table + + " WHERE probe_id = 1")) { + if (!result.next() || !"updated".equals(result.getString(1))) { + throw new SQLException("Temporary probe value was not readable"); + } + } + executeUpdate(statement, "DELETE FROM " + table + " WHERE probe_id = 1", deadline); + execute(statement, "DROP TABLE " + table, deadline); + } + } + + void dropIfExists(Connection connection, String table, DeadlineGuard deadline) throws SQLException { + try (Statement statement = connection.createStatement()) { + execute(statement, "DROP TABLE IF EXISTS " + table, deadline); + } + } + + void configureConnection(Connection connection, long deadline) throws SQLException { + connection.setNetworkTimeout(Runnable::run, timeoutMillis(deadline)); + } + + void rollback(Connection connection) { + try { + connection.rollback(); + } catch (SQLException ignored) { + // Preserve the stable validation error. + } + } + + private boolean utf8Compatible(Connection connection, MetadataDatabaseKind kind, + DeadlineGuard deadline) throws SQLException { + String sql = switch (kind) { + case MYSQL -> "SELECT @@character_set_database"; + case POSTGRESQL -> "SHOW server_encoding"; + case H2 -> null; + }; + if (sql == null) { + return true; + } + try (Statement statement = connection.createStatement()) { + prepare(statement, deadline.activeDeadline()); + try (ResultSet result = statement.executeQuery(sql)) { + return result.next() && result.getString(1).toLowerCase(Locale.ROOT).startsWith("utf8"); + } + } + } + + private void execute(Statement statement, String sql, DeadlineGuard deadline) throws SQLException { + prepare(statement, deadline.activeDeadline()); + statement.execute(sql); + } + + private void executeUpdate(Statement statement, String sql, DeadlineGuard deadline) throws SQLException { + prepare(statement, deadline.activeDeadline()); + statement.executeUpdate(sql); + } + + private void prepare(Statement statement, long deadline) throws SQLException { + statement.setQueryTimeout(timeoutSeconds(deadline)); + } + + private int timeoutMillis(long deadline) throws SQLException { + long remaining = remainingNanos(deadline); + return Math.toIntExact(Math.min(Integer.MAX_VALUE, + Math.max(1, TimeUnit.NANOSECONDS.toMillis(remaining)))); + } + + private int timeoutSeconds(long deadline) throws SQLException { + long remaining = remainingNanos(deadline); + long nanosPerSecond = TimeUnit.SECONDS.toNanos(1); + long roundedUp = remaining / nanosPerSecond + (remaining % nanosPerSecond == 0 ? 0 : 1); + return Math.toIntExact(Math.min(Integer.MAX_VALUE, roundedUp)); + } + + private long remainingNanos(long deadline) throws SQLException { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) { + throw new SQLException("Metadata probe deadline reached", "57014"); + } + return remaining; + } + + static long deadlineAfter(long durationNanos) { + long now = System.nanoTime(); + return durationNanos > Long.MAX_VALUE - now ? Long.MAX_VALUE : now + durationNanos; + } + + @FunctionalInterface + interface DeadlineGuard { + long activeDeadline() throws SQLException; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataProbeSession.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataProbeSession.java new file mode 100644 index 0000000000..ba4fcb8f79 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataProbeSession.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.sql.SQLException; +import java.time.Duration; +import java.util.Arrays; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Owns the deadline, cancellation, connection, candidate, and cleanup lifecycle of one probe. */ +final class JdbcMetadataProbeSession { + private static final String TABLE_PREFIX = "HZB_SETUP_PROBE_"; + private static final Optional CONNECTION_FAILED = + Optional.of(SetupErrorCode.METADATA_CONNECTION_FAILED); + private final Object stateLock = new Object(); + private final MetadataConnectionProbe.Request request; + private final long timeoutNanos; + private final ThreadPoolExecutor executor; + private final JdbcMetadataConnectionProbe.JdbcConnector connector; + private final JdbcMetadataProbeOperations operations = new JdbcMetadataProbeOperations(); + private final JdbcMetadataProbeCleanup cleanup; + private final long deadlineNanos; + private boolean cancelled; + private String candidateTable; + private boolean candidateCleanupConfirmed; + + JdbcMetadataProbeSession(MetadataConnectionProbe.Request request, Duration timeout, + ThreadPoolExecutor executor, Executor cleanupExecutor, + JdbcMetadataConnectionProbe.JdbcConnector connector) { + this.request = request; + this.timeoutNanos = Math.max(1, timeout.toNanos()); + this.executor = executor; + this.connector = connector; + this.cleanup = new JdbcMetadataProbeCleanup(request, timeoutNanos, cleanupExecutor, connector); + this.deadlineNanos = JdbcMetadataProbeOperations.deadlineAfter(timeoutNanos); + } + + Optional probe() { + Future> submitted; + try { + submitted = executor.submit(this::execute); + } catch (RejectedExecutionException overload) { + return CONNECTION_FAILED; + } + try { + return submitted.get(timeoutMillis(), TimeUnit.MILLISECONDS); + } catch (InterruptedException failure) { + cancel(submitted); + Thread.currentThread().interrupt(); + return CONNECTION_FAILED; + } catch (ExecutionException | TimeoutException failure) { + cancel(submitted); + return CONNECTION_FAILED; + } + } + + private Optional execute() { + char[] password = request.password().copy(); + try { + return validatePrimary(password); + } finally { + Thread.interrupted(); + scheduleCleanup(password); + Arrays.fill(password, '\0'); + } + } + + private Optional validatePrimary(char[] password) { + Connection connection = null; + try { + connection = connector.connect(request.jdbcUrl(), request.username(), password); + if (!acceptPrimary()) { + close(connection); + return CONNECTION_FAILED; + } + Connection claimed = connection; + try (claimed) { + operations.configureConnection(claimed, activeDeadline()); + Optional compatibility = operations.validateCompatibility( + claimed, request.kind(), this::activeDeadline); + return compatibility.isPresent() ? compatibility : validatePrivileges(claimed); + } + } catch (SQLException | RuntimeException failure) { + return CONNECTION_FAILED; + } + } + + private Optional validatePrivileges(Connection connection) { + try { + connection.setAutoCommit(false); + String table = registerCandidate(); + // MySQL DDL can commit implicitly, so the exact candidate is owned before CREATE is sent. + operations.executeCrudProbe(connection, table, this::activeDeadline); + connection.commit(); + confirmCandidateCleanup(table); + return Optional.empty(); + } catch (SQLException failure) { + operations.rollback(connection); + dropOnPrimary(connection); + return Optional.of(isStopped() ? SetupErrorCode.METADATA_CONNECTION_FAILED + : JdbcMetadataProbeFailureClassifier.classify(failure)); + } + } + + private void dropOnPrimary(Connection connection) { + String table = candidateNeedingCleanup(); + if (table == null || isStopped()) { + return; + } + try { + operations.dropIfExists(connection, table, this::activeDeadline); + connection.commit(); + confirmCandidateCleanup(table); + } catch (SQLException ignored) { + // Independent cleanup runs after the primary connection exits. + } + } + + private void scheduleCleanup(char[] password) { + String table = candidateNeedingCleanup(); + if (table != null) { + cleanup.schedule(table, password, () -> confirmCandidateCleanup(table)); + } + } + + private void cancel(Future> submitted) { + synchronized (stateLock) { + cancelled = true; + } + submitted.cancel(true); + } + + private boolean acceptPrimary() { + synchronized (stateLock) { + return !stoppedLocked(); + } + } + + private String registerCandidate() throws SQLException { + synchronized (stateLock) { + if (stoppedLocked()) { + throw deadlineFailure(); + } + candidateTable = TABLE_PREFIX + UUID.randomUUID().toString().replace("-", ""); + return candidateTable; + } + } + + private String candidateNeedingCleanup() { + synchronized (stateLock) { + return candidateCleanupConfirmed ? null : candidateTable; + } + } + + private void confirmCandidateCleanup(String table) { + synchronized (stateLock) { + if (table.equals(candidateTable)) { + candidateCleanupConfirmed = true; + } + } + } + + private boolean isStopped() { + synchronized (stateLock) { + return stoppedLocked(); + } + } + + private boolean stoppedLocked() { + return cancelled || System.nanoTime() >= deadlineNanos; + } + + private long activeDeadline() throws SQLException { + synchronized (stateLock) { + if (stoppedLocked()) { + throw deadlineFailure(); + } + return deadlineNanos; + } + } + + private long timeoutMillis() { + return Math.max(1, TimeUnit.NANOSECONDS.toMillis(timeoutNanos)); + } + + private static SQLException deadlineFailure() { + return new SQLException("Metadata probe deadline reached", "57014"); + } + + private static void close(Connection connection) { + if (connection != null) { + try { + connection.close(); + } catch (SQLException ignored) { + // Preserve the stable timeout or interruption error. + } + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataConnectionProbeCleanupTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataConnectionProbeCleanupTest.java new file mode 100644 index 0000000000..2c4b9ed447 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataConnectionProbeCleanupTest.java @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.startsWith; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLNonTransientConnectionException; +import java.sql.SQLTimeoutException; +import java.sql.Statement; +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.slf4j.LoggerFactory; + +class JdbcMetadataConnectionProbeCleanupTest { + + @Test + void timeoutAfterCreateUsesIndependentConnectionForExactCandidateCleanup() throws Exception { + ThreadPoolExecutor executor = executor(); + Connection primary = mock(Connection.class); + Connection cleanup = mock(Connection.class); + Statement primaryStatement = mock(Statement.class); + Statement cleanupStatement = mock(Statement.class); + CountDownLatch createStarted = new CountDownLatch(1); + CountDownLatch primaryClosed = new CountDownLatch(1); + CountDownLatch cleanupFinished = new CountDownLatch(1); + AtomicReference createSql = new AtomicReference<>(); + AtomicReference cleanupSql = new AtomicReference<>(); + AtomicInteger connections = new AtomicInteger(); + stubH2Compatibility(primary); + when(primary.createStatement()).thenReturn(primaryStatement); + when(cleanup.createStatement()).thenReturn(cleanupStatement); + doAnswer(ignored -> { + primaryClosed.countDown(); + return null; + }).when(primary).close(); + when(primaryStatement.execute(startsWith("CREATE TABLE"))).thenAnswer(invocation -> { + createSql.set(invocation.getArgument(0)); + createStarted.countDown(); + primaryClosed.await(2, TimeUnit.SECONDS); + throw new SQLException("closed after create", "08006", 51); + }); + when(cleanupStatement.execute(startsWith("DROP TABLE IF EXISTS"))).thenAnswer(invocation -> { + cleanupSql.set(invocation.getArgument(0)); + cleanupFinished.countDown(); + return true; + }); + JdbcMetadataConnectionProbe probe = new JdbcMetadataConnectionProbe( + Duration.ofMillis(250), executor, (url, username, password) -> + connections.getAndIncrement() == 0 ? primary : cleanup); + try { + assertThat(probe.probe(configuration("jdbc:h2:mem:cleanup"))) + .contains(SetupErrorCode.METADATA_CONNECTION_FAILED); + assertThat(createStarted.await(1, TimeUnit.SECONDS)).isTrue(); + assertThat(cleanupFinished.await(2, TimeUnit.SECONDS)).isTrue(); + assertThat(connections).hasValue(2); + String table = createSql.get().split(" ")[2]; + assertThat(cleanupSql.get()).isEqualTo("DROP TABLE IF EXISTS " + table); + verify(cleanup).setNetworkTimeout(any(), org.mockito.ArgumentMatchers.intThat(value -> value > 0)); + verify(cleanupStatement).setQueryTimeout(org.mockito.ArgumentMatchers.intThat(value -> value > 0)); + } finally { + primaryClosed.countDown(); + executor.shutdownNow(); + } + } + + @Test + void genuineCreatePermissionFailureRemainsInsufficientPrivileges() throws Exception { + assertThat(probeCreateFailure(new SQLException("permission denied", "42501", 70))) + .contains(SetupErrorCode.METADATA_INSUFFICIENT_PRIVILEGES); + } + + @Test + void connectionSqlStateWithoutCallerCancellationMapsToConnectionFailed() throws Exception { + assertThat(probeCreateFailure(new SQLException("connection lost", "08006", 71))) + .contains(SetupErrorCode.METADATA_CONNECTION_FAILED); + } + + @Test + void sqlTimeoutWithoutCallerCancellationMapsToConnectionFailed() throws Exception { + assertThat(probeCreateFailure(new SQLTimeoutException("timed out", "HYT00", 72))) + .contains(SetupErrorCode.METADATA_CONNECTION_FAILED); + } + + @Test + void nonTransientConnectionFailureWithoutSqlStateMapsToConnectionFailed() throws Exception { + assertThat(probeCreateFailure(new SQLNonTransientConnectionException("connection rejected", null, 73))) + .contains(SetupErrorCode.METADATA_CONNECTION_FAILED); + } + + @ParameterizedTest + @ValueSource(strings = {"HYT00", "HYT01", "57014"}) + void timeoutSqlStateMapsToConnectionFailed(String sqlState) throws Exception { + assertThat(probeCreateFailure(new SQLException("operation timed out", sqlState, 74))) + .contains(SetupErrorCode.METADATA_CONNECTION_FAILED); + } + + @Test + void cleanupFailureLogContainsOnlyFixedDiagnostics() throws Exception { + Logger logger = (Logger) LoggerFactory.getLogger(JdbcMetadataConnectionProbe.class); + CountDownLatch warningLogged = new CountDownLatch(1); + ListAppender appender = new ListAppender<>() { + @Override + protected void append(ILoggingEvent event) { + super.append(event); + if (event.getLevel() == Level.WARN) { + warningLogged.countDown(); + } + } + }; + appender.start(); + logger.addAppender(appender); + ThreadPoolExecutor executor = executor(); + Connection primary = mock(Connection.class); + Connection cleanup = mock(Connection.class); + Statement primaryStatement = mock(Statement.class); + Statement cleanupStatement = mock(Statement.class); + CountDownLatch primaryClosed = new CountDownLatch(1); + AtomicInteger connections = new AtomicInteger(); + stubH2Compatibility(primary); + when(primary.createStatement()).thenReturn(primaryStatement); + when(cleanup.createStatement()).thenReturn(cleanupStatement); + doAnswer(ignored -> { + primaryClosed.countDown(); + return null; + }).when(primary).close(); + when(primaryStatement.execute(startsWith("CREATE TABLE"))).thenAnswer(invocation -> { + primaryClosed.await(2, TimeUnit.SECONDS); + throw new SQLException("password=primary-secret", "08006", 61); + }); + when(cleanupStatement.execute(startsWith("DROP TABLE IF EXISTS"))).thenThrow( + new SQLException("password=cleanup-secret jdbc:h2:/private DROP TABLE", "42501", 77)); + JdbcMetadataConnectionProbe probe = new JdbcMetadataConnectionProbe( + Duration.ofMillis(200), executor, (url, username, password) -> + connections.getAndIncrement() == 0 ? primary : cleanup); + try { + assertThat(probe.probe(configuration("jdbc:h2:/private/config"))) + .contains(SetupErrorCode.METADATA_CONNECTION_FAILED); + verify(cleanupStatement, timeout(2_000)).execute(startsWith("DROP TABLE IF EXISTS")); + assertThat(warningLogged.await(1, TimeUnit.SECONDS)).isTrue(); + ILoggingEvent warning = appender.list.stream() + .filter(event -> event.getLevel() == Level.WARN).findFirst().orElseThrow(); + assertThat(warning.getFormattedMessage()) + .startsWith("Metadata probe cleanup failure kind=H2 table=HZB_SETUP_PROBE_") + .endsWith(" sqlState=42501 vendorCode=77") + .doesNotContain("password", "jdbc", "/private", "DROP TABLE", "cleanup-secret"); + assertThat(warning.getThrowableProxy()).isNull(); + } finally { + logger.detachAppender(appender); + appender.stop(); + primaryClosed.countDown(); + executor.shutdownNow(); + } + } + + private static MetadataDatabaseConfiguration configuration(String url) { + return new MetadataDatabaseConfiguration(MetadataDatabaseKind.H2, url, "sa", "password"); + } + + private static Optional probeCreateFailure(SQLException failure) throws Exception { + ThreadPoolExecutor executor = executor(); + Connection primary = mock(Connection.class); + Statement statement = mock(Statement.class); + stubH2Compatibility(primary); + when(primary.createStatement()).thenReturn(statement); + when(statement.execute(startsWith("CREATE TABLE"))).thenThrow(failure); + try { + return new JdbcMetadataConnectionProbe( + Duration.ofSeconds(2), executor, (url, username, password) -> primary) + .probe(configuration("jdbc:h2:mem:failure-classification")); + } finally { + executor.shutdownNow(); + } + } + + private static ThreadPoolExecutor executor() { + return new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(4), Thread.ofPlatform().name("probe-test-", 0).factory(), + new ThreadPoolExecutor.AbortPolicy()); + } + + private static void stubH2Compatibility(Connection connection) throws SQLException { + DatabaseMetaData metadata = mock(DatabaseMetaData.class); + when(connection.getMetaData()).thenReturn(metadata); + when(metadata.getDatabaseProductName()).thenReturn("H2"); + when(metadata.getSchemas()).thenReturn(mock(ResultSet.class)); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataConnectionProbeConcurrencyTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataConnectionProbeConcurrencyTest.java new file mode 100644 index 0000000000..a00c049917 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataConnectionProbeConcurrencyTest.java @@ -0,0 +1,282 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.startsWith; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.junit.jupiter.api.Test; + +class JdbcMetadataConnectionProbeConcurrencyTest { + + @Test + void lateConnectionAfterTimeoutIsClosedBeforeCompatibilityOrDdl() throws Exception { + ThreadPoolExecutor executor = executor(); + CountDownLatch connecting = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + Connection late = mock(Connection.class); + JdbcMetadataConnectionProbe probe = new JdbcMetadataConnectionProbe( + Duration.ofMillis(100), executor, (url, username, password) -> { + connecting.countDown(); + while (release.getCount() > 0) { + try { + release.await(); + } catch (InterruptedException ignored) { + // Emulate a driver that ignores interruption while connecting. + } + } + return late; + }); + try { + assertThat(probe.probe(configuration("jdbc:h2:mem:late"))) + .contains(SetupErrorCode.METADATA_CONNECTION_FAILED); + assertThat(connecting.await(1, TimeUnit.SECONDS)).isTrue(); + release.countDown(); + verify(late, timeout(2_000)).close(); + verify(late, never()).getMetaData(); + verify(late, never()).createStatement(); + } finally { + release.countDown(); + executor.shutdownNow(); + } + } + + @Test + void interruptedCallerCancelsFutureAndClosesActiveConnection() throws Exception { + ThreadPoolExecutor executor = executor(); + Connection primary = mock(Connection.class); + Connection cleanup = mock(Connection.class); + Statement primaryStatement = mock(Statement.class); + Statement cleanupStatement = mock(Statement.class); + CountDownLatch createStarted = new CountDownLatch(1); + CountDownLatch workerClosed = new CountDownLatch(1); + AtomicReference> outcome = new AtomicReference<>(); + AtomicBoolean interruptRestored = new AtomicBoolean(); + AtomicInteger connections = new AtomicInteger(); + stubH2Compatibility(primary); + when(primary.createStatement()).thenReturn(primaryStatement); + when(cleanup.createStatement()).thenReturn(cleanupStatement); + when(primaryStatement.execute(startsWith("CREATE TABLE"))).thenAnswer(invocation -> { + createStarted.countDown(); + try { + new CountDownLatch(1).await(); + return true; + } catch (InterruptedException cancelled) { + throw new SQLException("cancelled", "08006", 52); + } + }); + doAnswer(ignored -> { + workerClosed.countDown(); + return null; + }).when(primary).close(); + JdbcMetadataConnectionProbe probe = new JdbcMetadataConnectionProbe( + Duration.ofSeconds(5), executor, (url, username, password) -> + connections.getAndIncrement() == 0 ? primary : cleanup); + Thread caller = Thread.ofPlatform().start(() -> { + outcome.set(probe.probe(configuration("jdbc:h2:mem:interrupted"))); + interruptRestored.set(Thread.currentThread().isInterrupted()); + }); + try { + assertThat(createStarted.await(2, TimeUnit.SECONDS)).isTrue(); + caller.interrupt(); + caller.join(2_000); + assertThat(caller.isAlive()).isFalse(); + assertThat(outcome.get()).contains(SetupErrorCode.METADATA_CONNECTION_FAILED); + assertThat(interruptRestored).isTrue(); + assertThat(workerClosed.await(1, TimeUnit.SECONDS)).isTrue(); + verify(primary, timeout(1_000).times(1)).close(); + } finally { + caller.interrupt(); + caller.join(1_000); + executor.shutdownNow(); + } + } + + @Test + void timeoutReturnsBeforeWorkerOwnedCloseAndCleanupWaitsForClose() throws Exception { + ThreadPoolExecutor executor = executor(); + Connection primary = mock(Connection.class); + Connection cleanup = mock(Connection.class); + Statement primaryStatement = mock(Statement.class); + Statement cleanupStatement = mock(Statement.class); + CountDownLatch createStarted = new CountDownLatch(1); + CountDownLatch neverReleaseStatement = new CountDownLatch(1); + CountDownLatch closeStarted = new CountDownLatch(1); + CountDownLatch releaseClose = new CountDownLatch(1); + CountDownLatch cleanupConnecting = new CountDownLatch(1); + AtomicInteger connections = new AtomicInteger(); + AtomicReference> outcome = new AtomicReference<>(); + stubH2Compatibility(primary); + when(primary.createStatement()).thenReturn(primaryStatement); + when(cleanup.createStatement()).thenReturn(cleanupStatement); + when(primaryStatement.execute(startsWith("CREATE TABLE"))).thenAnswer(invocation -> { + createStarted.countDown(); + try { + neverReleaseStatement.await(); + return true; + } catch (InterruptedException cancelled) { + throw new SQLException("cancelled", "08006", 82); + } + }); + doAnswer(ignored -> { + closeStarted.countDown(); + while (releaseClose.getCount() > 0) { + try { + releaseClose.await(); + } catch (InterruptedException ignoredInterrupt) { + // The worker remains the close owner until the driver returns. + } + } + return null; + }).when(primary).close(); + JdbcMetadataConnectionProbe probe = new JdbcMetadataConnectionProbe( + Duration.ofMillis(100), executor, (url, username, password) -> { + if (connections.getAndIncrement() == 0) { + return primary; + } + cleanupConnecting.countDown(); + return cleanup; + }); + Thread caller = Thread.ofPlatform().start(() -> + outcome.set(probe.probe(configuration("jdbc:h2:mem:blocking-close")))); + try { + assertThat(createStarted.await(1, TimeUnit.SECONDS)).isTrue(); + assertThat(closeStarted.await(1, TimeUnit.SECONDS)).isTrue(); + caller.join(500); + assertThat(caller.isAlive()).isFalse(); + assertThat(cleanupConnecting.getCount()).isEqualTo(1); + releaseClose.countDown(); + assertThat(cleanupConnecting.await(1, TimeUnit.SECONDS)).isTrue(); + assertThat(outcome.get()).contains(SetupErrorCode.METADATA_CONNECTION_FAILED); + verify(primary, timeout(1_000).times(1)).close(); + } finally { + neverReleaseStatement.countDown(); + releaseClose.countDown(); + caller.interrupt(); + caller.join(1_000); + executor.shutdownNow(); + } + } + + @Test + void cleanupConnectorThatIgnoresInterruptDoesNotConsumePrimaryProbeCapacity() throws Exception { + ThreadPoolExecutor executor = executor(); + ThreadPoolExecutor cleanupExecutor = executor(); + Connection firstPrimary = mock(Connection.class); + Connection cleanup = mock(Connection.class); + Connection secondPrimary = mock(Connection.class); + Statement firstStatement = mock(Statement.class); + Statement cleanupStatement = mock(Statement.class); + CountDownLatch firstClosed = new CountDownLatch(1); + CountDownLatch cleanupConnecting = new CountDownLatch(1); + CountDownLatch releaseCleanup = new CountDownLatch(1); + CountDownLatch secondConnecting = new CountDownLatch(1); + AtomicInteger connections = new AtomicInteger(); + stubH2Compatibility(firstPrimary); + when(firstPrimary.createStatement()).thenReturn(firstStatement); + when(cleanup.createStatement()).thenReturn(cleanupStatement); + DatabaseMetaData mismatch = mock(DatabaseMetaData.class); + when(secondPrimary.getMetaData()).thenReturn(mismatch); + when(mismatch.getDatabaseProductName()).thenReturn("PostgreSQL"); + doAnswer(ignored -> { + firstClosed.countDown(); + return null; + }).when(firstPrimary).close(); + when(firstStatement.execute(startsWith("CREATE TABLE"))).thenAnswer(invocation -> { + firstClosed.await(2, TimeUnit.SECONDS); + throw new SQLException("closed", "08006", 81); + }); + JdbcMetadataConnectionProbe.JdbcConnector connector = (url, username, password) -> { + int connection = connections.getAndIncrement(); + if (connection == 0) { + return firstPrimary; + } + if (connection == 1) { + cleanupConnecting.countDown(); + while (releaseCleanup.getCount() > 0) { + try { + releaseCleanup.await(); + } catch (InterruptedException ignored) { + // Emulate a cleanup connection attempt with no portable hard cancellation. + } + } + return cleanup; + } + secondConnecting.countDown(); + return secondPrimary; + }; + try { + JdbcMetadataConnectionProbe first = new JdbcMetadataConnectionProbe( + Duration.ofMillis(200), executor, cleanupExecutor, connector); + assertThat(first.probe(configuration("jdbc:h2:mem:blocked-cleanup"))) + .contains(SetupErrorCode.METADATA_CONNECTION_FAILED); + assertThat(cleanupConnecting.await(2, TimeUnit.SECONDS)).isTrue(); + + JdbcMetadataConnectionProbe second = new JdbcMetadataConnectionProbe( + Duration.ofSeconds(1), executor, cleanupExecutor, connector); + assertThat(second.probe(configuration("jdbc:h2:mem:second-probe"))) + .contains(SetupErrorCode.METADATA_SCHEMA_MISMATCH); + assertThat(secondConnecting.await(1, TimeUnit.SECONDS)).isTrue(); + } finally { + firstClosed.countDown(); + releaseCleanup.countDown(); + executor.shutdownNow(); + cleanupExecutor.shutdownNow(); + } + } + + private static MetadataDatabaseConfiguration configuration(String url) { + return new MetadataDatabaseConfiguration(MetadataDatabaseKind.H2, url, "sa", "password"); + } + + private static ThreadPoolExecutor executor() { + return new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(4), Thread.ofPlatform().name("probe-test-", 0).factory(), + new ThreadPoolExecutor.AbortPolicy()); + } + + private static void stubH2Compatibility(Connection connection) throws SQLException { + DatabaseMetaData metadata = mock(DatabaseMetaData.class); + when(connection.getMetaData()).thenReturn(metadata); + when(metadata.getDatabaseProductName()).thenReturn("H2"); + when(metadata.getSchemas()).thenReturn(mock(ResultSet.class)); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataConnectionProbeTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataConnectionProbeTest.java index 07123e7077..d703a1b306 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataConnectionProbeTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataConnectionProbeTest.java @@ -18,13 +18,19 @@ package org.apache.hertzbeat.manager.setup.workflow; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.contains; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.sql.Connection; +import java.sql.DatabaseMetaData; import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; import java.time.Duration; import java.util.ArrayList; @@ -34,6 +40,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; @@ -46,11 +53,23 @@ class JdbcMetadataConnectionProbeTest { String url = "jdbc:h2:mem:setup_validation;DB_CLOSE_DELAY=-1"; var configuration = new MetadataDatabaseConfiguration( MetadataDatabaseKind.H2, url, "sa", "password"); + AtomicInteger connections = new AtomicInteger(); + ThreadPoolExecutor executor = executor(); - assertThat(new JdbcMetadataConnectionProbe(Duration.ofSeconds(30)).probe(configuration)).isEmpty(); - try (var connection = java.sql.DriverManager.getConnection(url, "sa", "password"); - var result = connection.getMetaData().getTables(null, null, "HZB_SETUP_PROBE_%", null)) { - assertThat(result.next()).isFalse(); + try { + JdbcMetadataConnectionProbe probe = new JdbcMetadataConnectionProbe( + Duration.ofSeconds(30), executor, (jdbcUrl, username, password) -> { + connections.incrementAndGet(); + return java.sql.DriverManager.getConnection(jdbcUrl, username, new String(password)); + }); + assertThat(probe.probe(configuration)).isEmpty(); + assertThat(connections).hasValue(1); + try (var connection = java.sql.DriverManager.getConnection(url, "sa", "password"); + var result = connection.getMetaData().getTables(null, null, "HZB_SETUP_PROBE_%", null)) { + assertThat(result.next()).isFalse(); + } + } finally { + executor.shutdownNow(); } } @@ -98,21 +117,50 @@ class JdbcMetadataConnectionProbeTest { @Test void successfulTransactionalDdlDropsBeforeCommitWithoutRollback() throws Exception { + ThreadPoolExecutor executor = executor(); + JdbcMetadataConnectionProbe.JdbcConnector connector = mock(JdbcMetadataConnectionProbe.JdbcConnector.class); Connection connection = mock(Connection.class); Statement statement = mock(Statement.class); ResultSet result = mock(ResultSet.class); + stubH2Compatibility(connection); when(connection.createStatement()).thenReturn(statement); when(statement.executeQuery(contains("SELECT probe_value"))).thenReturn(result); when(result.next()).thenReturn(true); when(result.getString(1)).thenReturn("updated"); + when(connector.connect(anyString(), anyString(), any(char[].class))).thenReturn(connection); - assertThat(new JdbcMetadataConnectionProbe(Duration.ofSeconds(1)) - .validatePrivileges(connection)).isEmpty(); + try { + assertThat(new JdbcMetadataConnectionProbe(Duration.ofSeconds(1), executor, connector) + .probe(configuration("jdbc:h2:mem:mock-success"))).isEmpty(); - var order = inOrder(statement, connection); - order.verify(statement).execute(contains("CREATE TABLE")); - order.verify(statement).execute(contains("DROP TABLE")); - order.verify(connection).commit(); - org.mockito.Mockito.verify(connection, org.mockito.Mockito.never()).rollback(); + var order = inOrder(statement, connection); + order.verify(statement).execute(contains("CREATE TABLE")); + order.verify(statement).execute(contains("DROP TABLE")); + order.verify(connection).commit(); + verify(connection, never()).rollback(); + verify(connection).setNetworkTimeout(any(), org.mockito.ArgumentMatchers.intThat(value -> value > 0)); + verify(statement, org.mockito.Mockito.atLeast(1)).setQueryTimeout( + org.mockito.ArgumentMatchers.intThat(value -> value > 0)); + verify(connector).connect(anyString(), anyString(), any(char[].class)); + } finally { + executor.shutdownNow(); + } + } + + private static MetadataDatabaseConfiguration configuration(String url) { + return new MetadataDatabaseConfiguration(MetadataDatabaseKind.H2, url, "sa", "password"); + } + + private static ThreadPoolExecutor executor() { + return new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(4), Thread.ofPlatform().name("probe-test-", 0).factory(), + new ThreadPoolExecutor.AbortPolicy()); + } + + private static void stubH2Compatibility(Connection connection) throws SQLException { + DatabaseMetaData metadata = mock(DatabaseMetaData.class); + when(connection.getMetaData()).thenReturn(metadata); + when(metadata.getDatabaseProductName()).thenReturn("H2"); + when(metadata.getSchemas()).thenReturn(mock(ResultSet.class)); } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataProbeCleanupBoundaryTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataProbeCleanupBoundaryTest.java new file mode 100644 index 0000000000..c8e2e61842 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataProbeCleanupBoundaryTest.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.slf4j.LoggerFactory; + +class JdbcMetadataProbeCleanupBoundaryTest { + + @Test + void occupiedZeroQueueCleanupRejectsSecondCandidateWithoutAnotherConnection() throws Exception { + assertThat(((ThreadPoolExecutor) JdbcMetadataProbeCleanup.sharedExecutor()).getQueue().remainingCapacity()) + .isZero(); + ThreadPoolExecutor cleanupExecutor = cleanupExecutor(); + CountDownLatch connecting = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicInteger connections = new AtomicInteger(); + Connection connection = mock(Connection.class); + when(connection.createStatement()).thenReturn(mock(Statement.class)); + Logger logger = (Logger) LoggerFactory.getLogger(JdbcMetadataConnectionProbe.class); + ListAppender appender = appender(logger); + try (var request = request("jdbc:h2:mem:zero-queue")) { + JdbcMetadataProbeCleanup cleanup = new JdbcMetadataProbeCleanup( + request, Duration.ofSeconds(1).toNanos(), cleanupExecutor, (url, username, password) -> { + connections.incrementAndGet(); + connecting.countDown(); + while (release.getCount() > 0) { + try { + release.await(); + } catch (InterruptedException ignored) { + // Emulate a cleanup connector that ignores interruption. + } + } + return connection; + }); + cleanup.schedule("HZB_SETUP_PROBE_FIRST", "first-secret".toCharArray(), () -> { }); + assertThat(connecting.await(1, TimeUnit.SECONDS)).isTrue(); + cleanup.schedule("HZB_SETUP_PROBE_SECOND", "second-secret".toCharArray(), () -> { }); + + assertThat(connections).hasValue(1); + ILoggingEvent warning = appender.list.stream() + .filter(event -> event.getLevel() == Level.WARN).findFirst().orElseThrow(); + assertThat(warning.getFormattedMessage()) + .isEqualTo("Metadata probe cleanup failure kind=H2 table=HZB_SETUP_PROBE_SECOND " + + "sqlState=unknown vendorCode=0") + .doesNotContain("secret"); + } finally { + release.countDown(); + logger.detachAppender(appender); + appender.stop(); + cleanupExecutor.shutdownNow(); + } + } + + @ParameterizedTest + @ValueSource(strings = {"cleanupsecret", "é0806", "ab123", "08-01"}) + void invalidSqlStateIsLoggedAsUnknown(String sqlState) throws Exception { + Logger logger = (Logger) LoggerFactory.getLogger(JdbcMetadataConnectionProbe.class); + ListAppender appender = appender(logger); + try (var request = request("jdbc:h2:mem:invalid-state")) { + JdbcMetadataProbeCleanup cleanup = new JdbcMetadataProbeCleanup( + request, Duration.ofSeconds(1).toNanos(), Runnable::run, + (url, username, password) -> { + throw new SQLException("exception-secret", sqlState, 91); + }); + cleanup.schedule("HZB_SETUP_PROBE_STATE", "password-secret".toCharArray(), () -> { }); + + ILoggingEvent warning = appender.list.getFirst(); + assertThat(warning.getFormattedMessage()) + .isEqualTo("Metadata probe cleanup failure kind=H2 table=HZB_SETUP_PROBE_STATE " + + "sqlState=unknown vendorCode=91") + .doesNotContain(sqlState, "exception-secret", "password-secret"); + } finally { + logger.detachAppender(appender); + appender.stop(); + } + } + + private static MetadataConnectionProbe.Request request(String url) { + return new MetadataConnectionProbe.Request( + MetadataDatabaseKind.H2, url, "sa", SecretValue.of("password")); + } + + private static ThreadPoolExecutor cleanupExecutor() { + return new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, + new SynchronousQueue<>(), Thread.ofPlatform().name("cleanup-test-", 0).factory(), + new ThreadPoolExecutor.AbortPolicy()); + } + + private static ListAppender appender(Logger logger) { + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + return appender; + } +} From 2d6d146fc5064116878d646fcece88ecd8470729 Mon Sep 17 00:00:00 2001 From: Logic Date: Sun, 9 Aug 2026 04:18:12 +0800 Subject: [PATCH 19/71] Preserve safe startup failure diagnostics --- .../runtime/HertzBeatStartupCoordinator.java | 16 +- .../runtime/StartupFailureReporter.java | 78 +++++++ .../runtime/StartupFailureReporterTest.java | 202 ++++++++++++++++++ 3 files changed, 295 insertions(+), 1 deletion(-) create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupFailureReporter.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupFailureReporterTest.java diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java index 29766e312c..0d82c96862 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java @@ -26,12 +26,19 @@ public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition private final StartupDecisionProbe probe; private final StartupContextLauncher launcher; + private final StartupFailureReporter failureReporter; private String[] args = new String[0]; private RunningApplicationContext currentContext; public HertzBeatStartupCoordinator(StartupDecisionProbe probe, StartupContextLauncher launcher) { + this(probe, launcher, new StartupFailureReporter()); + } + + HertzBeatStartupCoordinator( + StartupDecisionProbe probe, StartupContextLauncher launcher, StartupFailureReporter failureReporter) { this.probe = Objects.requireNonNull(probe, "probe"); this.launcher = Objects.requireNonNull(launcher, "launcher"); + this.failureReporter = Objects.requireNonNull(failureReporter, "failureReporter"); } public synchronized RunningApplicationContext start(String[] applicationArgs) { @@ -40,6 +47,7 @@ public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition try { decision = Objects.requireNonNull(probe.probe(args.clone()), "startup decision"); } catch (RuntimeException exception) { + failureReporter.report(StartupFailureReporter.Stage.STARTUP_PROBE, RuntimeMode.RECOVERY, exception); decision = StartupDecision.recovery(); } return transition(decision); @@ -65,12 +73,18 @@ public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition currentContext = launch(decision); } catch (RuntimeException launchFailure) { if (decision.mode() == RuntimeMode.RECOVERY) { + failureReporter.report(StartupFailureReporter.Stage.RECOVERY_LAUNCH, RuntimeMode.RECOVERY, launchFailure); throw launchFailure; } + failureReporter.report(StartupFailureReporter.Stage.CONTEXT_LAUNCH, decision.mode(), launchFailure); try { currentContext = launch(StartupDecision.recovery()); } catch (RuntimeException recoveryFailure) { - recoveryFailure.addSuppressed(launchFailure); + failureReporter.report( + StartupFailureReporter.Stage.RECOVERY_LAUNCH, RuntimeMode.RECOVERY, recoveryFailure); + if (recoveryFailure != launchFailure) { + recoveryFailure.addSuppressed(launchFailure); + } throw recoveryFailure; } } diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupFailureReporter.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupFailureReporter.java new file mode 100644 index 0000000000..2c4677d495 --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupFailureReporter.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup.runtime; + +import java.util.Objects; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Emits startup diagnostics without retaining exception messages or causes. */ +final class StartupFailureReporter { + + private static final Logger LOGGER = LoggerFactory.getLogger(StartupFailureReporter.class); + private final DiagnosticSink sink; + + StartupFailureReporter() { + this((stage, mode, exceptionClass) -> LOGGER.warn( + "Startup failure stage={} mode={} exception={}", stage, mode, exceptionClass)); + } + + StartupFailureReporter(DiagnosticSink sink) { + this.sink = Objects.requireNonNull(sink, "sink"); + } + + void report(Stage stage, RuntimeMode mode, RuntimeException failure) { + try { + sink.report(stage.value(), safeMode(mode), failure.getClass().getName()); + } catch (RuntimeException ignored) { + // Diagnostics are best-effort and must never change startup recovery control flow. + } + } + + private static String safeMode(RuntimeMode mode) { + return switch (mode) { + case SETUP_ONLY -> "setup_only"; + case FULL_SETUP_GATED -> "full_setup_gated"; + case NORMAL -> "normal"; + case RECOVERY -> "recovery"; + }; + } + + @FunctionalInterface + interface DiagnosticSink { + + void report(String stage, String mode, String exceptionClass); + } + + enum Stage { + STARTUP_PROBE("startup-probe"), + CONTEXT_LAUNCH("context-launch"), + RECOVERY_LAUNCH("recovery-launch"); + + private final String value; + + Stage(String value) { + this.value = value; + } + + String value() { + return value; + } + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupFailureReporterTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupFailureReporterTest.java new file mode 100644 index 0000000000..8a8d50a81b --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupFailureReporterTest.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup.runtime; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import java.util.List; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.slf4j.LoggerFactory; + +class StartupFailureReporterTest { + + private static final String SECRET = "password=change-me"; + private static final String JDBC_URL = "jdbc:postgresql://database.internal/hertzbeat"; + private static final String CLI_SECRET = "--spring.datasource.password=command-line-secret"; + + private final Logger logger = (Logger) LoggerFactory.getLogger(StartupFailureReporter.class); + private final ListAppender appender = new ListAppender<>(); + + @BeforeEach + void attachAppender() { + appender.start(); + logger.addAppender(appender); + } + + @AfterEach + void detachAppender() { + logger.detachAppender(appender); + appender.stop(); + } + + @Test + void probeFailureFallsBackToRecoveryWithSafeDiagnostic() { + RuntimeException probeFailure = new IllegalStateException(SECRET + " " + JDBC_URL); + probeFailure.setStackTrace(new StackTraceElement[]{new StackTraceElement( + "secret." + SECRET, "connect." + JDBC_URL, CLI_SECRET, 7)}); + RecordingLauncher launcher = new RecordingLauncher(); + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( + ignored -> { + throw probeFailure; + }, launcher); + + RunningApplicationContext context = coordinator.start(new String[]{CLI_SECRET}); + + assertEquals(RuntimeMode.RECOVERY, context.mode()); + assertEquals(List.of(RuntimeMode.RECOVERY), launcher.attempts); + assertSafeDiagnostic(appender.list.getFirst(), "startup-probe", RuntimeMode.RECOVERY, probeFailure); + } + + @ParameterizedTest + @EnumSource(value = RuntimeMode.class, names = {"NORMAL", "FULL_SETUP_GATED"}) + void contextFailureRetainsSafeDiagnosticWhenRecoverySucceeds(RuntimeMode failedMode) { + RuntimeException launchFailure = new IllegalArgumentException(SECRET + " config=" + JDBC_URL); + RecordingLauncher launcher = new RecordingLauncher(); + launcher.failure(failedMode, launchFailure); + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( + ignored -> new StartupDecision(failedMode), launcher); + + RunningApplicationContext context = coordinator.start(new String[]{CLI_SECRET}); + + assertEquals(RuntimeMode.RECOVERY, context.mode()); + assertEquals(List.of(failedMode, RuntimeMode.RECOVERY), launcher.attempts); + assertSafeDiagnostic(appender.list.getFirst(), "context-launch", failedMode, launchFailure); + } + + @Test + void recoveryFailurePreservesPropagationAndSuppressionWithoutDiagnosticLeaks() { + RuntimeException launchFailure = new IllegalStateException(SECRET + " " + CLI_SECRET); + RuntimeException recoveryFailure = new UnsupportedOperationException(JDBC_URL + " " + SECRET); + RecordingLauncher launcher = new RecordingLauncher(); + launcher.failure(RuntimeMode.NORMAL, launchFailure); + launcher.failure(RuntimeMode.RECOVERY, recoveryFailure); + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( + ignored -> StartupDecision.normal(), launcher); + + RuntimeException thrown = assertThrows(RuntimeException.class, + () -> coordinator.start(new String[]{CLI_SECRET})); + + assertSame(recoveryFailure, thrown); + assertEquals(1, thrown.getSuppressed().length); + assertSame(launchFailure, thrown.getSuppressed()[0]); + assertEquals(2, appender.list.size()); + assertSafeDiagnostic(appender.list.get(0), "context-launch", RuntimeMode.NORMAL, launchFailure); + assertSafeDiagnostic(appender.list.get(1), "recovery-launch", RuntimeMode.RECOVERY, recoveryFailure); + } + + @Test + void diagnosticSinkFailureCannotPreventFailClosedRecovery() { + RecordingLauncher launcher = new RecordingLauncher(); + StartupFailureReporter reporter = new StartupFailureReporter((stage, mode, exceptionClass) -> { + throw new IllegalStateException("diagnostic sink unavailable " + SECRET); + }); + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( + ignored -> { + throw new IllegalArgumentException(JDBC_URL); + }, launcher, reporter); + + RunningApplicationContext context = coordinator.start(new String[]{CLI_SECRET}); + + assertEquals(RuntimeMode.RECOVERY, context.mode()); + assertEquals(List.of(RuntimeMode.RECOVERY), launcher.attempts); + assertEquals(0, appender.list.size()); + } + + @Test + void identicalContextAndRecoveryFailureDoesNotAttemptSelfSuppression() { + RuntimeException sharedFailure = new IllegalStateException(SECRET); + RecordingLauncher launcher = new RecordingLauncher(); + launcher.failure(RuntimeMode.NORMAL, sharedFailure); + launcher.failure(RuntimeMode.RECOVERY, sharedFailure); + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( + ignored -> StartupDecision.normal(), launcher); + + RuntimeException thrown = assertThrows(RuntimeException.class, + () -> coordinator.start(new String[]{CLI_SECRET})); + + assertSame(sharedFailure, thrown); + assertEquals(0, thrown.getSuppressed().length); + assertEquals(2, appender.list.size()); + assertSafeDiagnostic(appender.list.get(0), "context-launch", RuntimeMode.NORMAL, sharedFailure); + assertSafeDiagnostic(appender.list.get(1), "recovery-launch", RuntimeMode.RECOVERY, sharedFailure); + } + + private static void assertSafeDiagnostic( + ILoggingEvent event, String stage, RuntimeMode mode, RuntimeException originalFailure) { + assertEquals("Startup failure stage=" + stage + " mode=" + mode.value() + " exception=" + + originalFailure.getClass().getName(), event.getFormattedMessage()); + assertFalse(event.getFormattedMessage().contains(SECRET)); + assertFalse(event.getFormattedMessage().contains(JDBC_URL)); + assertFalse(event.getFormattedMessage().contains(CLI_SECRET)); + assertEquals(3, event.getArgumentArray().length); + assertEquals(stage, event.getArgumentArray()[0]); + assertEquals(mode.value(), event.getArgumentArray()[1]); + assertEquals(originalFailure.getClass().getName(), event.getArgumentArray()[2]); + assertNull(event.getThrowableProxy()); + } + + private static final class RecordingLauncher implements StartupContextLauncher { + + private final List attempts = new java.util.ArrayList<>(); + private final java.util.Map failures = new java.util.EnumMap<>(RuntimeMode.class); + + void failure(RuntimeMode mode, RuntimeException failure) { + failures.put(mode, failure); + } + + @Override + public RunningApplicationContext launch( + StartupDecision decision, String[] args, + org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition transition) { + RuntimeMode mode = decision.mode(); + attempts.add(mode); + RuntimeException failure = failures.get(mode); + if (failure != null) { + throw failure; + } + return new RunningApplicationContext() { + @Override + public RuntimeMode mode() { + return mode; + } + + @Override + public boolean isActive() { + return true; + } + + @Override + public void close() { + // Nothing to release in this test context. + } + }; + } + } +} From a866b0f97b83ae260c65e432823efe339bb3cec9 Mon Sep 17 00:00:00 2001 From: Logic Date: Sun, 9 Aug 2026 05:12:17 +0800 Subject: [PATCH 20/71] Stream setup configuration exports --- .../manager/setup/api/SetupApiContract.java | 10 + .../manager/setup/api/SetupController.java | 8 +- .../config/ExternalConfigExportArtifact.java | 53 --- .../setup/config/SensitiveExportContent.java | 62 ---- .../setup/workflow/SetupExportRenderer.java | 199 +++++++---- .../setup/api/SetupApiContractTest.java | 17 + .../setup/api/SetupControllerTest.java | 56 +++ .../ExternalConfigExportArtifactTest.java | 65 ---- .../workflow/SetupExportRendererTest.java | 326 ++++++++++++++++++ 9 files changed, 552 insertions(+), 244 deletions(-) delete mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ExternalConfigExportArtifact.java delete mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SensitiveExportContent.java delete mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ExternalConfigExportArtifactTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupExportRendererTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java index 3e1ccf96a8..aa51c5a3fe 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java @@ -567,6 +567,16 @@ public final class SetupApiContract { /** Safe download metadata. Secret-bearing content is written only as a no-store attachment. */ public record ExportResponse(@NotBlank String fileName, @NotBlank String mediaType) { + + public ExportResponse { + if (fileName == null || !fileName.matches("[A-Za-z0-9._-]+")) { + throw new IllegalArgumentException("Export filename is unsafe"); + } + if (mediaType == null || mediaType.isBlank() || mediaType.indexOf('\\') >= 0 + || mediaType.indexOf('\r') >= 0 || mediaType.indexOf('\n') >= 0) { + throw new IllegalArgumentException("Export media type is unsafe"); + } + } } /** Setup completion acknowledgement. */ diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupController.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupController.java index 44e82e9c11..d054e929d7 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupController.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupController.java @@ -119,12 +119,12 @@ public class SetupController { @PostMapping(SetupApiContract.EXPORT_PATH) public ResponseEntity export(@Valid @RequestBody ExportRequest request) { var metadata = workflow.prepareExport(request); - var artifact = exportRenderer.render(request, metadata); - StreamingResponseBody body = output -> artifact.content().writeTo(output); + // Async response I/O failures propagate to the servlet container after attachment headers are committed. + StreamingResponseBody body = output -> exportRenderer.write(request, output); return SetupHttpContract.noStore() .header(HttpHeaders.CONTENT_DISPOSITION, - "attachment; filename=\"" + artifact.fileName() + "\"") - .header(HttpHeaders.CONTENT_TYPE, artifact.mediaType()).body(body); + "attachment; filename=\"" + metadata.fileName() + "\"") + .header(HttpHeaders.CONTENT_TYPE, metadata.mediaType()).body(body); } @PostMapping(SetupApiContract.COMPLETE_PATH) diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ExternalConfigExportArtifact.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ExternalConfigExportArtifact.java deleted file mode 100644 index e2267e71e1..0000000000 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ExternalConfigExportArtifact.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hertzbeat.manager.setup.config; - -import java.util.Objects; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; - -/** Safe attachment metadata and sensitive bytes for operator-applied configuration. */ -public record ExternalConfigExportArtifact( - String fileName, - String mediaType, - SensitiveExportContent content) { - - public ExternalConfigExportArtifact { - if (fileName == null || !fileName.matches("[A-Za-z0-9._-]+")) { - throw new IllegalArgumentException("Export filename is unsafe"); - } - if (mediaType == null || mediaType.isBlank() || mediaType.indexOf('\\') >= 0 - || mediaType.indexOf('\r') >= 0 || mediaType.indexOf('\n') >= 0) { - throw new IllegalArgumentException("Export media type is unsafe"); - } - Objects.requireNonNull(content, "content"); - } - - public SetupOperationState state() { - return SetupOperationState.AWAITING_EXTERNAL_APPLY; - } - - public boolean noStore() { - return true; - } - - @Override - public String toString() { - return "ExternalConfigExportArtifact[fileName=" + fileName + ", mediaType=" + mediaType - + ", content=, state=awaiting_external_apply, noStore=true]"; - } -} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SensitiveExportContent.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SensitiveExportContent.java deleted file mode 100644 index bbccd8a7f1..0000000000 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SensitiveExportContent.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hertzbeat.manager.setup.config; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.Arrays; - -/** Defensive secret-bearing export bytes that never render their content. */ -public final class SensitiveExportContent implements AutoCloseable { - - private final byte[] content; - - private SensitiveExportContent(byte[] content) { - this.content = content.clone(); - } - - public static SensitiveExportContent of(byte[] content) { - if (content == null || content.length == 0) { - throw new IllegalArgumentException("Export content must not be empty"); - } - return new SensitiveExportContent(content); - } - - public byte[] copy() { - return content.clone(); - } - - public synchronized void writeTo(OutputStream output) throws IOException { - try { - output.write(content); - output.flush(); - } finally { - close(); - } - } - - @Override - public synchronized void close() { - Arrays.fill(content, (byte) 0); - } - - @Override - public String toString() { - return "SensitiveExportContent[]"; - } -} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupExportRenderer.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupExportRenderer.java index f2326eb2d9..ce46689278 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupExportRenderer.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupExportRenderer.java @@ -26,93 +26,172 @@ import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.GREPTIME_PASSWORD; import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.GREPTIME_USERNAME; +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; import java.nio.charset.StandardCharsets; -import java.util.Arrays; import java.util.Base64; +import java.util.Objects; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportFormat; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportRequest; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportResponse; -import org.apache.hertzbeat.manager.setup.config.ExternalConfigExportArtifact; -import org.apache.hertzbeat.manager.setup.config.SensitiveExportContent; -/** Renders the frozen export formats entirely in memory without writing setup secrets to disk. */ +/** Incrementally writes frozen external configuration formats without retaining a rendered body. */ public final class SetupExportRenderer { - public ExternalConfigExportArtifact render(ExportRequest request, ExportResponse metadata) { - String content = switch (request.format()) { - case YAML -> yaml(request.configuration()); - case ENV -> environment(request.configuration()); - case KUBERNETES_SECRET -> kubernetesSecret(request.configuration()); - }; - byte[] bytes = content.getBytes(StandardCharsets.UTF_8); - try { - return new ExternalConfigExportArtifact(metadata.fileName(), metadata.mediaType(), - SensitiveExportContent.of(bytes)); - } finally { - Arrays.fill(bytes, (byte) 0); + public void write(ExportRequest request, OutputStream output) throws IOException { + Objects.requireNonNull(request, "request"); + Objects.requireNonNull(output, "output"); + switch (request.format()) { + case YAML -> writeYaml(request.configuration(), output); + case ENV -> writeEnvironment(request.configuration(), output); + case KUBERNETES_SECRET -> writeKubernetesSecret(request.configuration(), output); + default -> throw new IllegalArgumentException("Unsupported export format"); } + output.flush(); } - private static String yaml(ConfigurationRequest request) { + private static void writeYaml(ConfigurationRequest request, OutputStream output) throws IOException { var metadata = request.managementDatabase(); var telemetry = request.telemetryStore(); - StringBuilder output = new StringBuilder(); - yaml(output, DATASOURCE_URL, metadata.jdbcUrl()); - yaml(output, DATASOURCE_USERNAME, metadata.username()); - yaml(output, DATASOURCE_PASSWORD, metadata.password()); - yaml(output, GREPTIME_GRPC, telemetry.grpcEndpoints()); - yaml(output, GREPTIME_HTTP, telemetry.httpEndpoint()); - yaml(output, GREPTIME_DATABASE, telemetry.database()); + Writer writer = utf8Writer(output); + writeYamlEntry(writer, DATASOURCE_URL, metadata.jdbcUrl()); + writeYamlEntry(writer, DATASOURCE_USERNAME, metadata.username()); + writeYamlEntry(writer, DATASOURCE_PASSWORD, metadata.password()); + writeYamlEntry(writer, GREPTIME_GRPC, telemetry.grpcEndpoints()); + writeYamlEntry(writer, GREPTIME_HTTP, telemetry.httpEndpoint()); + writeYamlEntry(writer, GREPTIME_DATABASE, telemetry.database()); if (telemetry.username() != null) { - yaml(output, GREPTIME_USERNAME, telemetry.username()); - yaml(output, GREPTIME_PASSWORD, telemetry.password()); + writeYamlEntry(writer, GREPTIME_USERNAME, telemetry.username()); + writeYamlEntry(writer, GREPTIME_PASSWORD, telemetry.password()); } - return output.toString(); + writer.flush(); } - private static String environment(ConfigurationRequest request) { + private static void writeEnvironment(ConfigurationRequest request, OutputStream output) throws IOException { var metadata = request.managementDatabase(); var telemetry = request.telemetryStore(); - StringBuilder output = new StringBuilder(); - env(output, "SPRING_DATASOURCE_URL", metadata.jdbcUrl()); - env(output, "SPRING_DATASOURCE_USERNAME", metadata.username()); - env(output, "SPRING_DATASOURCE_PASSWORD", metadata.password()); - env(output, "WAREHOUSE_STORE_GREPTIME_GRPC_ENDPOINTS", telemetry.grpcEndpoints()); - env(output, "WAREHOUSE_STORE_GREPTIME_HTTP_ENDPOINT", telemetry.httpEndpoint()); - env(output, "WAREHOUSE_STORE_GREPTIME_DATABASE", telemetry.database()); + Writer writer = utf8Writer(output); + writeEnvironmentEntry(writer, "SPRING_DATASOURCE_URL", metadata.jdbcUrl()); + writeEnvironmentEntry(writer, "SPRING_DATASOURCE_USERNAME", metadata.username()); + writeEnvironmentEntry(writer, "SPRING_DATASOURCE_PASSWORD", metadata.password()); + writeEnvironmentEntry(writer, "WAREHOUSE_STORE_GREPTIME_GRPC_ENDPOINTS", telemetry.grpcEndpoints()); + writeEnvironmentEntry(writer, "WAREHOUSE_STORE_GREPTIME_HTTP_ENDPOINT", telemetry.httpEndpoint()); + writeEnvironmentEntry(writer, "WAREHOUSE_STORE_GREPTIME_DATABASE", telemetry.database()); if (telemetry.username() != null) { - env(output, "WAREHOUSE_STORE_GREPTIME_USERNAME", telemetry.username()); - env(output, "WAREHOUSE_STORE_GREPTIME_PASSWORD", telemetry.password()); + writeEnvironmentEntry(writer, "WAREHOUSE_STORE_GREPTIME_USERNAME", telemetry.username()); + writeEnvironmentEntry(writer, "WAREHOUSE_STORE_GREPTIME_PASSWORD", telemetry.password()); } - return output.toString(); + writer.flush(); } - private static String kubernetesSecret(ConfigurationRequest request) { - StringBuilder output = new StringBuilder("apiVersion: v1\nkind: Secret\nmetadata:\n" - + " name: hertzbeat-setup\ntype: Opaque\ndata:\n"); - data(output, "managed-application.yml", yaml(request)); - data(output, "managed-setup.env", environment(request)); - return output.toString(); + private static void writeKubernetesSecret(ConfigurationRequest request, OutputStream output) throws IOException { + Writer writer = utf8Writer(output); + writer.write("apiVersion: v1\nkind: Secret\nmetadata:\n name: hertzbeat-setup\ntype: Opaque\ndata:\n"); + writer.write(" managed-application.yml: "); + writer.flush(); + writeBase64(request, ExportFormat.YAML, output); + writer.write("\n managed-setup.env: "); + writer.flush(); + writeBase64(request, ExportFormat.ENV, output); + writer.write('\n'); + writer.flush(); } - private static void yaml(StringBuilder output, String key, String value) { - output.append(key).append(": '").append(value.replace("'", "''")).append("'\n"); - } - - private static void env(StringBuilder output, String key, String value) { - output.append(key).append('=').append(environmentValue(value)).append('\n'); - } - - private static String environmentValue(String value) { - if (value.matches("[A-Za-z0-9_./:@+\\-]*")) { - return value; + private static void writeBase64( + ConfigurationRequest request, ExportFormat format, OutputStream output) throws IOException { + OutputStream encoded = Base64.getEncoder().wrap(new CloseShieldOutputStream(output)); + if (format == ExportFormat.YAML) { + writeYaml(request, encoded); + } else { + writeEnvironment(request, encoded); } - return "'" + value.replace("'", "'\"'\"'") + "'"; + encoded.close(); } - private static void data(StringBuilder output, String key, String value) { - output.append(" ").append(key).append(": ").append(Base64.getEncoder() - .encodeToString(value.getBytes(StandardCharsets.UTF_8))).append('\n'); + private static void writeYamlEntry(Writer writer, String key, String value) throws IOException { + writer.write(key); + writer.write(": '"); + writeDynamicValue(writer, value, "''"); + writer.write("'\n"); } + private static void writeEnvironmentEntry(Writer writer, String key, String value) throws IOException { + writer.write(key); + writer.write('='); + if (isSafeEnvironmentValue(value)) { + writeDynamicValue(writer, value, null); + } else { + writer.write('\''); + writeDynamicValue(writer, value, "'\"'\"'"); + writer.write('\''); + } + writer.write('\n'); + } + + private static void writeDynamicValue( + Writer writer, String value, String apostropheEscape) throws IOException { + // Setup export is a one-time path; per-code-point flushes bound secret residency in encoder buffers. + for (int index = 0; index < value.length();) { + char current = value.charAt(index); + if (current == '\'' && apostropheEscape != null) { + writer.write(apostropheEscape); + index++; + } else if (Character.isHighSurrogate(current) && index + 1 < value.length() + && Character.isLowSurrogate(value.charAt(index + 1))) { + writer.write(value, index, 2); + index += 2; + } else if (Character.isSurrogate(current)) { + writer.write('?'); + index++; + } else { + writer.write(current); + index++; + } + writer.flush(); + } + } + + private static boolean isSafeEnvironmentValue(String value) { + for (int index = 0; index < value.length(); index++) { + char current = value.charAt(index); + if (!(current >= 'A' && current <= 'Z') && !(current >= 'a' && current <= 'z') + && !(current >= '0' && current <= '9') && "_./:@+-".indexOf(current) < 0) { + return false; + } + } + return true; + } + + private static Writer utf8Writer(OutputStream output) { + return new OutputStreamWriter(new DeferredFlushOutputStream(output), StandardCharsets.UTF_8); + } + + /** Drains the UTF-8 encoder without turning every secret code point into a servlet flush. */ + private static final class DeferredFlushOutputStream extends FilterOutputStream { + + private DeferredFlushOutputStream(OutputStream output) { + super(output); + } + + @Override + public void flush() { + // The top-level renderer owns the single caller-visible flush. + } + } + + /** Lets a Base64 wrapper finalize padding without owning the caller's response stream. */ + private static final class CloseShieldOutputStream extends FilterOutputStream { + + private CloseShieldOutputStream(OutputStream output) { + super(output); + } + + @Override + public void close() { + // Base64 padding is already written before close reaches this shield. + } + } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java index c08ab602d8..e2e10c698d 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java @@ -30,6 +30,7 @@ import java.util.List; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.AdministratorRequest; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigSource; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; @@ -119,6 +120,22 @@ class SetupApiContractTest { assertComponents(SetupApiContract.CompleteResponse.class, "phase", "completedAt", "loginPath", "username"); } + @Test + void exportMetadataRejectsUnsafeAttachmentHeaders() { + assertThrows(IllegalArgumentException.class, + () -> new ExportResponse("../managed.env", "text/plain")); + assertThrows(IllegalArgumentException.class, + () -> new ExportResponse("managed/env", "text/plain")); + assertThrows(IllegalArgumentException.class, + () -> new ExportResponse("managed\\env", "text/plain")); + assertThrows(IllegalArgumentException.class, + () -> new ExportResponse("managed\r\nenv", "text/plain")); + assertThrows(IllegalArgumentException.class, + () -> new ExportResponse("managed.env", "text/plain\r\nx-test: value")); + assertThrows(IllegalArgumentException.class, + () -> new ExportResponse("managed.env", "text\\plain")); + } + @Test void secretInputsAreWriteOnlyAndSafeToRender() throws Exception { MetadataDatabaseConfiguration metadata = new MetadataDatabaseConfiguration( diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupControllerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupControllerTest.java index fa533600ff..6ebd1ac553 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupControllerTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupControllerTest.java @@ -17,8 +17,14 @@ package org.apache.hertzbeat.manager.setup.api; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.asyncDispatch; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; @@ -29,11 +35,18 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.request; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; import java.time.Instant; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigSource; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportFormat; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportRequest; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ManagementDatabaseSummary; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionalConfigurationSummary; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupAccess; @@ -41,6 +54,7 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.StatusResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreSummary; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.UnlockResponse; import org.apache.hertzbeat.manager.setup.security.SetupUnlockRejected; @@ -163,6 +177,39 @@ class SetupControllerTest { "SPRING_DATASOURCE_PASSWORD=database-secret"))); } + @Test + void exportRenderingStartsOnlyWhenStreamingBodyExecutes() throws Exception { + SetupExportRenderer renderer = mock(SetupExportRenderer.class); + SetupController controller = new SetupController(workflow, + mock(SetupHttpUnlockService.class), mock(SetupResponseTransition.class), renderer); + ExportRequest request = exportRequest(); + when(workflow.prepareExport(request)).thenReturn(new ExportResponse("hertzbeat-setup.env", "text/plain")); + + var response = controller.export(request); + + verifyNoInteractions(renderer); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + response.getBody().writeTo(output); + verify(renderer).write(same(request), same(output)); + } + + @Test + void clientDisconnectPropagatesFromStreamingCallback() throws Exception { + SetupExportRenderer renderer = mock(SetupExportRenderer.class); + SetupController controller = new SetupController(workflow, + mock(SetupHttpUnlockService.class), mock(SetupResponseTransition.class), renderer); + ExportRequest request = exportRequest(); + IOException clientAbort = new IOException("client disconnected with database-secret"); + when(workflow.prepareExport(request)).thenReturn(new ExportResponse("hertzbeat-setup.env", "text/plain")); + doThrow(clientAbort).when(renderer).write(same(request), any(OutputStream.class)); + var response = controller.export(request); + + IOException thrown = assertThrows(IOException.class, + () -> response.getBody().writeTo(new ByteArrayOutputStream())); + + assertSame(clientAbort, thrown); + } + @Test void unexpectedSetupFailureUsesStableNoStoreEnvelope() throws Exception { when(workflow.status()).thenThrow(new IllegalStateException("database-secret")); @@ -184,4 +231,13 @@ class SetupControllerTest { .andExpect(status().isConflict()) .andExpect(jsonPath("$.errorCode").value("config_read_only")); } + + private static ExportRequest exportRequest() { + return new ExportRequest(ExportFormat.ENV, + new ConfigurationRequest(SetupPhase.CONFIGURATION_REQUIRED, ApplyMode.EXTERNAL_APPLY, + new MetadataDatabaseConfiguration(MetadataDatabaseKind.H2, + "jdbc:h2:mem:setup", "sa", "database-secret"), + new TelemetryStoreConfiguration(TelemetryStoreKind.GREPTIME, + "localhost:4001", "http://localhost:4000", "public", null, null))); + } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ExternalConfigExportArtifactTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ExternalConfigExportArtifactTest.java deleted file mode 100644 index cdfe35f26e..0000000000 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ExternalConfigExportArtifactTest.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hertzbeat.manager.setup.config; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.nio.charset.StandardCharsets; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; -import org.junit.jupiter.api.Test; - -class ExternalConfigExportArtifactTest { - - private static final String SECRET = "export-secret-value"; - - @Test - void carriesSensitiveContentWithoutClaimingItWasApplied() { - byte[] source = SECRET.getBytes(StandardCharsets.UTF_8); - ExternalConfigExportArtifact artifact = new ExternalConfigExportArtifact( - "hertzbeat-managed.env", "text/plain", SensitiveExportContent.of(source)); - source[0] = 'x'; - - assertArrayEquals(SECRET.getBytes(StandardCharsets.UTF_8), artifact.content().copy()); - assertEquals(SetupOperationState.AWAITING_EXTERNAL_APPLY, artifact.state()); - assertTrue(artifact.noStore()); - assertFalse(artifact.toString().contains(SECRET)); - assertFalse(artifact.content().toString().contains(SECRET)); - } - - @Test - void rejectsUnsafeAttachmentNames() { - SensitiveExportContent content = SensitiveExportContent.of("safe".getBytes(StandardCharsets.UTF_8)); - - assertThrows(IllegalArgumentException.class, - () -> new ExternalConfigExportArtifact("../managed.env", "text/plain", content)); - assertThrows(IllegalArgumentException.class, - () -> new ExternalConfigExportArtifact("managed/env", "text/plain", content)); - assertThrows(IllegalArgumentException.class, - () -> new ExternalConfigExportArtifact("managed\\env", "text/plain", content)); - assertThrows(IllegalArgumentException.class, - () -> new ExternalConfigExportArtifact("managed\r\nenv", "text/plain", content)); - assertThrows(IllegalArgumentException.class, - () -> new ExternalConfigExportArtifact("managed.env", "text/plain\r\nx-test: value", content)); - assertThrows(IllegalArgumentException.class, - () -> new ExternalConfigExportArtifact("managed.env", "text\\plain", content)); - } -} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupExportRendererTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupExportRendererTest.java new file mode 100644 index 0000000000..49b581e744 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupExportRendererTest.java @@ -0,0 +1,326 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportFormat; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreKind; +import org.junit.jupiter.api.Test; + +class SetupExportRendererTest { + + private final SetupExportRenderer renderer = new SetupExportRenderer(); + + @Test + void streamsYamlAndEnvironmentWithExactFrozenEscaping() throws IOException { + assertEquals(expectedYaml(), render(ExportFormat.YAML).toString(StandardCharsets.UTF_8)); + assertEquals(expectedEnvironment(), render(ExportFormat.ENV).toString(StandardCharsets.UTF_8)); + } + + @Test + void preservesSurrogateReplacementAndCodePointEscaping() throws IOException { + String password = "pair-😀-high-\uD800-low-\uDC00-O'Brien"; + ConfigurationRequest configuration = configurationWithPassword(password); + + String yaml = render(ExportFormat.YAML, configuration).toString(StandardCharsets.UTF_8); + String environment = render(ExportFormat.ENV, configuration).toString(StandardCharsets.UTF_8); + + assertTrue(yaml.contains("spring.datasource.password: 'pair-😀-high-?-low-?-O''Brien'\n")); + assertTrue(environment.contains( + "SPRING_DATASOURCE_PASSWORD='pair-😀-high-?-low-?-O'\"'\"'Brien'\n")); + } + + @Test + void longSecretReachesOutputBeforeTheFieldTraversalCanComplete() throws Exception { + String password = "x".repeat(9_000); + String emittedThroughFirstSecretCodePoint = """ + spring.datasource.url: 'jdbc:h2:mem:setup' + spring.datasource.username: 'safe_user' + spring.datasource.password: 'x"""; + int blockAfter = emittedThroughFirstSecretCodePoint.getBytes(StandardCharsets.UTF_8).length; + BlockingOutputStream output = new BlockingOutputStream(blockAfter); + + try (var executor = Executors.newSingleThreadExecutor()) { + var rendering = executor.submit(() -> { + renderer.write(new ExportRequest( + ExportFormat.YAML, configurationWithPassword(password)), output); + return null; + }); + assertTrue(output.firstSecretWrite.await(5, TimeUnit.SECONDS)); + try { + assertEquals(blockAfter, output.blockedSize); + assertFalse(rendering.isDone()); + } finally { + output.release.countDown(); + } + rendering.get(5, TimeUnit.SECONDS); + } + assertEquals(1, output.flushCount); + + CloseTrackingOutputStream environmentOutput = new CloseTrackingOutputStream(); + renderer.write(new ExportRequest( + ExportFormat.ENV, configurationWithPassword(password)), environmentOutput); + assertEquals(1, environmentOutput.flushCount); + } + + @Test + void streamsKubernetesPayloadsAsExactBase64WithoutClosingCallerOutput() throws IOException { + byte[] yaml = render(ExportFormat.YAML).toByteArray(); + byte[] environment = render(ExportFormat.ENV).toByteArray(); + CloseTrackingOutputStream output = new CloseTrackingOutputStream(); + + renderer.write(new ExportRequest(ExportFormat.KUBERNETES_SECRET, configuration()), output); + output.write('!'); + + String manifest = output.toString(StandardCharsets.UTF_8); + String encodedYaml = Base64.getEncoder().encodeToString(yaml); + String encodedEnvironment = Base64.getEncoder().encodeToString(environment); + assertFalse(output.closed); + assertEquals(1, output.flushCount); + assertTrue(encodedYaml.endsWith("==")); + assertEquals("apiVersion: v1\nkind: Secret\nmetadata:\n name: hertzbeat-setup\n" + + "type: Opaque\ndata:\n managed-application.yml: " + + encodedYaml + "\n managed-setup.env: " + encodedEnvironment + "\n!", + manifest); + assertArrayEquals(yaml, decodePayload(manifest, "managed-application.yml")); + assertArrayEquals(environment, decodePayload(manifest, "managed-setup.env")); + } + + @Test + void propagatesClientWriteFailureWithoutMaterializingFallbackContent() { + IOException clientAbort = new IOException("client disconnected with export-secret"); + OutputStream output = new OutputStream() { + @Override + public void write(int value) throws IOException { + throw clientAbort; + } + + @Override + public void write(byte[] value, int offset, int length) throws IOException { + throw clientAbort; + } + }; + + IOException thrown = assertThrows(IOException.class, + () -> renderer.write(new ExportRequest(ExportFormat.YAML, configuration()), output)); + + assertSame(clientAbort, thrown); + } + + @Test + void kubernetesStreamingPropagatesMidPayloadFailureWithoutClosingCallerOutput() { + IOException clientAbort = new IOException("client disconnected with export-secret"); + FailingCloseTrackingOutputStream output = new FailingCloseTrackingOutputStream(116, clientAbort); + + IOException thrown = assertThrows(IOException.class, + () -> renderer.write( + new ExportRequest(ExportFormat.KUBERNETES_SECRET, configuration()), output)); + + assertSame(clientAbort, thrown); + assertFalse(output.closed); + } + + private ByteArrayOutputStream render(ExportFormat format) throws IOException { + return render(format, configuration()); + } + + private ByteArrayOutputStream render( + ExportFormat format, ConfigurationRequest configuration) throws IOException { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + renderer.write(new ExportRequest(format, configuration), output); + return output; + } + + private static byte[] decodePayload(String manifest, String key) { + String prefix = " " + key + ": "; + for (String line : manifest.split("\n")) { + if (line.startsWith(prefix)) { + return Base64.getDecoder().decode(line.substring(prefix.length())); + } + } + throw new AssertionError("Missing Kubernetes Secret payload"); + } + + private static ConfigurationRequest configuration() { + return new ConfigurationRequest(SetupPhase.CONFIGURATION_REQUIRED, ApplyMode.EXTERNAL_APPLY, + new MetadataDatabaseConfiguration(MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://db/main?token=${DB}\nline", "safe_user", "O'Brien"), + new TelemetryStoreConfiguration(TelemetryStoreKind.GREPTIME, + "", "https://münich.example/δ", "ordinary", + "telemetry", "line1\n${TOKEN}'λ")); + } + + private static ConfigurationRequest configurationWithPassword(String password) { + return new ConfigurationRequest(SetupPhase.CONFIGURATION_REQUIRED, ApplyMode.EXTERNAL_APPLY, + new MetadataDatabaseConfiguration(MetadataDatabaseKind.H2, + "jdbc:h2:mem:setup", "safe_user", password), + new TelemetryStoreConfiguration(TelemetryStoreKind.GREPTIME, + "localhost:4001", "http://localhost:4000", "public", null, null)); + } + + private static String expectedYaml() { + return """ + spring.datasource.url: 'jdbc:postgresql://db/main?token=${DB} + line' + spring.datasource.username: 'safe_user' + spring.datasource.password: 'O''Brien' + warehouse.store.greptime.grpc-endpoints: '' + warehouse.store.greptime.http-endpoint: 'https://münich.example/δ' + warehouse.store.greptime.database: 'ordinary' + warehouse.store.greptime.username: 'telemetry' + warehouse.store.greptime.password: 'line1 + ${TOKEN}''λ' + """; + } + + private static String expectedEnvironment() { + return """ + SPRING_DATASOURCE_URL='jdbc:postgresql://db/main?token=${DB} + line' + SPRING_DATASOURCE_USERNAME=safe_user + SPRING_DATASOURCE_PASSWORD='O'"'"'Brien' + WAREHOUSE_STORE_GREPTIME_GRPC_ENDPOINTS= + WAREHOUSE_STORE_GREPTIME_HTTP_ENDPOINT='https://münich.example/δ' + WAREHOUSE_STORE_GREPTIME_DATABASE=ordinary + WAREHOUSE_STORE_GREPTIME_USERNAME=telemetry + WAREHOUSE_STORE_GREPTIME_PASSWORD='line1 + ${TOKEN}'"'"'λ' + """; + } + + private static final class CloseTrackingOutputStream extends ByteArrayOutputStream { + + private boolean closed; + private int flushCount; + + @Override + public void flush() throws IOException { + flushCount++; + super.flush(); + } + + @Override + public void close() throws IOException { + closed = true; + super.close(); + } + } + + private static final class FailingCloseTrackingOutputStream extends OutputStream { + + private final int failAfter; + private final IOException failure; + private int written; + private boolean closed; + + private FailingCloseTrackingOutputStream(int failAfter, IOException failure) { + this.failAfter = failAfter; + this.failure = failure; + } + + @Override + public void write(byte[] value, int offset, int length) throws IOException { + int allowed = Math.max(0, failAfter - written); + if (allowed < length) { + written += allowed; + throw failure; + } + written += length; + } + + @Override + public void write(int value) throws IOException { + if (written >= failAfter) { + throw failure; + } + written++; + } + + @Override + public void close() { + closed = true; + } + } + + private static final class BlockingOutputStream extends OutputStream { + + private final int blockAfter; + private final CountDownLatch firstSecretWrite = new CountDownLatch(1); + private final CountDownLatch release = new CountDownLatch(1); + private int written; + private volatile int blockedSize; + private int flushCount; + + private BlockingOutputStream(int blockAfter) { + this.blockAfter = blockAfter; + } + + @Override + public void write(byte[] value, int offset, int length) throws IOException { + written += length; + blockIfFirstSecretCodePointWasWritten(); + } + + @Override + public void write(int value) throws IOException { + written++; + blockIfFirstSecretCodePointWasWritten(); + } + + @Override + public void flush() { + flushCount++; + } + + private void blockIfFirstSecretCodePointWasWritten() throws IOException { + if (blockedSize == 0 && written >= blockAfter) { + blockedSize = written; + firstSecretWrite.countDown(); + try { + if (!release.await(5, TimeUnit.SECONDS)) { + throw new IOException("Timed out waiting to continue export rendering"); + } + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while observing export streaming", exception); + } + } + } + } +} From 865fcc6a9dab49a1837a80857f6f5813f52b19b6 Mon Sep 17 00:00:00 2001 From: Logic Date: Sun, 9 Aug 2026 05:38:36 +0800 Subject: [PATCH 21/71] Harden setup failure diagnostics --- .../setup/api/SetupExceptionHandler.java | 28 +++++++--- .../setup/api/SetupFailureDiagnosticSink.java | 14 +++++ .../LoggingRecoveryFailureReporter.java | 10 +--- .../config/ManagedConfigurationRecovery.java | 12 ++--- .../ManagedConfigurationTransaction.java | 4 +- .../setup/config/RecoveryFailureReporter.java | 11 +++- .../api/SetupExceptionHandlerLoggingTest.java | 44 +++++++++++---- .../LoggingRecoveryFailureReporterTest.java | 53 ++++++++++++++----- .../ManagedConfigurationTransactionTest.java | 44 +++++++++++++-- 9 files changed, 168 insertions(+), 52 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupFailureDiagnosticSink.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandler.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandler.java index 56adc5cfa8..80b1405323 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandler.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandler.java @@ -18,6 +18,7 @@ package org.apache.hertzbeat.manager.setup.api; import java.time.Clock; +import java.util.Objects; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; import org.apache.hertzbeat.manager.setup.security.SetupUnlockRejected; import org.apache.hertzbeat.manager.setup.workflow.SetupWorkflowConflict; @@ -36,13 +37,19 @@ public class SetupExceptionHandler { private static final Logger LOGGER = LoggerFactory.getLogger(SetupExceptionHandler.class); private final Clock clock; + private final SetupFailureDiagnosticSink diagnosticSink; public SetupExceptionHandler() { - this(Clock.systemUTC()); + this(Clock.systemUTC(), SetupExceptionHandler::logUnexpectedFailure); } SetupExceptionHandler(Clock clock) { - this.clock = clock; + this(clock, SetupExceptionHandler::logUnexpectedFailure); + } + + SetupExceptionHandler(Clock clock, SetupFailureDiagnosticSink diagnosticSink) { + this.clock = Objects.requireNonNull(clock, "clock"); + this.diagnosticSink = Objects.requireNonNull(diagnosticSink, "diagnosticSink"); } @ExceptionHandler(SetupApiException.class) @@ -71,15 +78,20 @@ public class SetupExceptionHandler { @ExceptionHandler(Exception.class) public ResponseEntity unexpectedFailure(Exception failure) { - LOGGER.error("Unexpected setup request failure exception={}", - failure.getClass().getName(), diagnosticCopy(failure)); + reportSafely(failure.getClass().getName()); return response(HttpStatus.INTERNAL_SERVER_ERROR, SetupErrorCode.INTERNAL_ERROR); } - private static Throwable diagnosticCopy(Throwable failure) { - Throwable diagnostic = new Throwable(); - diagnostic.setStackTrace(failure.getStackTrace()); - return diagnostic; + private static void logUnexpectedFailure(String exceptionClass) { + LOGGER.error("Unexpected setup request failure exception={}", exceptionClass); + } + + private void reportSafely(String exceptionClass) { + try { + diagnosticSink.report(exceptionClass); + } catch (RuntimeException ignored) { + // Diagnostics must never replace the stable HTTP failure response. + } } private ResponseEntity response(HttpStatus status, SetupErrorCode code) { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupFailureDiagnosticSink.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupFailureDiagnosticSink.java new file mode 100644 index 0000000000..70c0a8ae73 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupFailureDiagnosticSink.java @@ -0,0 +1,14 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.api; + +/** Receives the only safe, fixed diagnostic field for an unexpected setup failure. */ +@FunctionalInterface +interface SetupFailureDiagnosticSink { + void report(String exceptionClass); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/LoggingRecoveryFailureReporter.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/LoggingRecoveryFailureReporter.java index 73f0421dda..330997f224 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/LoggingRecoveryFailureReporter.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/LoggingRecoveryFailureReporter.java @@ -15,14 +15,8 @@ final class LoggingRecoveryFailureReporter implements RecoveryFailureReporter { private static final Logger LOGGER = LoggerFactory.getLogger(LoggingRecoveryFailureReporter.class); @Override - public void report(Stage stage, Store store, Exception failure) { + public void report(Stage stage, Store store, String exceptionClass) { LOGGER.warn("Managed configuration recovery failure stage={} store={} exception={}", - stage, store, failure.getClass().getName(), diagnosticCopy(failure)); - } - - private static Throwable diagnosticCopy(Throwable failure) { - Throwable diagnostic = new Throwable(); - diagnostic.setStackTrace(failure.getStackTrace()); - return diagnostic; + stage, store, exceptionClass); } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationRecovery.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationRecovery.java index f0d540feca..5e21465863 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationRecovery.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationRecovery.java @@ -60,14 +60,14 @@ final class ManagedConfigurationRecovery { try { applicationStore.discardCandidate(); } catch (IOException failure) { - reporter.report(RecoveryFailureReporter.Stage.DISCARD_CANDIDATE, + RecoveryFailureReporter.reportSafely(reporter, RecoveryFailureReporter.Stage.DISCARD_CANDIDATE, RecoveryFailureReporter.Store.APPLICATION, failure); discarded = false; } try { secretStore.discardCandidate(); } catch (IOException failure) { - reporter.report(RecoveryFailureReporter.Stage.DISCARD_CANDIDATE, + RecoveryFailureReporter.reportSafely(reporter, RecoveryFailureReporter.Stage.DISCARD_CANDIDATE, RecoveryFailureReporter.Store.SECRET, failure); discarded = false; } @@ -109,7 +109,7 @@ final class ManagedConfigurationRecovery { applicationStore.promoteCandidate(candidate.value().orElseThrow(), candidate.generation().orElseThrow()); return true; } catch (IOException failure) { - reporter.report(RecoveryFailureReporter.Stage.PROMOTE_CANDIDATE, + RecoveryFailureReporter.reportSafely(reporter, RecoveryFailureReporter.Stage.PROMOTE_CANDIDATE, RecoveryFailureReporter.Store.APPLICATION, failure); return false; } @@ -120,7 +120,7 @@ final class ManagedConfigurationRecovery { secretStore.promoteCandidate(candidate.value().orElseThrow(), candidate.generation().orElseThrow()); return true; } catch (IOException failure) { - reporter.report(RecoveryFailureReporter.Stage.PROMOTE_CANDIDATE, + RecoveryFailureReporter.reportSafely(reporter, RecoveryFailureReporter.Stage.PROMOTE_CANDIDATE, RecoveryFailureReporter.Store.SECRET, failure); return false; } @@ -131,7 +131,7 @@ final class ManagedConfigurationRecovery { applicationStore.restoreActive(candidate); return true; } catch (IOException failure) { - reporter.report(RecoveryFailureReporter.Stage.RESTORE_ACTIVE, + RecoveryFailureReporter.reportSafely(reporter, RecoveryFailureReporter.Stage.RESTORE_ACTIVE, RecoveryFailureReporter.Store.APPLICATION, failure); return false; } @@ -142,7 +142,7 @@ final class ManagedConfigurationRecovery { secretStore.restoreActive(candidate); return true; } catch (IOException failure) { - reporter.report(RecoveryFailureReporter.Stage.RESTORE_ACTIVE, + RecoveryFailureReporter.reportSafely(reporter, RecoveryFailureReporter.Stage.RESTORE_ACTIVE, RecoveryFailureReporter.Store.SECRET, failure); return false; } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransaction.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransaction.java index 1274e808f0..84dce63e6c 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransaction.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransaction.java @@ -115,14 +115,14 @@ public final class ManagedConfigurationTransaction { try { applicationStore.promoteCandidate(bundle.application(), generation); } catch (IOException failure) { - reporter.report(RecoveryFailureReporter.Stage.PROMOTE_CANDIDATE, + RecoveryFailureReporter.reportSafely(reporter, RecoveryFailureReporter.Stage.PROMOTE_CANDIDATE, RecoveryFailureReporter.Store.APPLICATION, failure); return recovery.rollback(previousApplication, previousSecrets); } try { secretStore.promoteCandidate(bundle.secrets(), generation); } catch (IOException failure) { - reporter.report(RecoveryFailureReporter.Stage.PROMOTE_CANDIDATE, + RecoveryFailureReporter.reportSafely(reporter, RecoveryFailureReporter.Stage.PROMOTE_CANDIDATE, RecoveryFailureReporter.Store.SECRET, failure); return recovery.rollback(previousApplication, previousSecrets); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/RecoveryFailureReporter.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/RecoveryFailureReporter.java index a926a6ce1b..beb2da3abc 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/RecoveryFailureReporter.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/RecoveryFailureReporter.java @@ -10,7 +10,16 @@ package org.apache.hertzbeat.manager.setup.config; /** Secret-free diagnostic boundary for managed configuration recovery failures. */ @FunctionalInterface public interface RecoveryFailureReporter { - void report(Stage stage, Store store, Exception failure); + void report(Stage stage, Store store, String exceptionClass); + + /** Reports diagnostics without allowing an adapter failure to change recovery control flow. */ + static void reportSafely(RecoveryFailureReporter reporter, Stage stage, Store store, Exception failure) { + try { + reporter.report(stage, store, failure.getClass().getName()); + } catch (RuntimeException ignored) { + // Recovery and rollback outcomes are authoritative over diagnostics. + } + } /** Recovery operation stage that failed. */ enum Stage { diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandlerLoggingTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandlerLoggingTest.java index 570ebacb8a..00d8d5e16f 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandlerLoggingTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandlerLoggingTest.java @@ -18,7 +18,8 @@ package org.apache.hertzbeat.manager.setup.api; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; @@ -34,6 +35,10 @@ import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.read.ListAppender; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.Arrays; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; import org.apache.hertzbeat.manager.setup.runtime.SetupResponseTransition; import org.apache.hertzbeat.manager.setup.security.SetupHttpUnlockService; @@ -70,8 +75,15 @@ class SetupExceptionHandlerLoggingTest { } @Test - void unexpectedFailureLogsFixedContextAndThrowableWhileResponseStaysSafe() throws Exception { + void unexpectedFailureLogsOnlyFixedContextWhileResponseStaysSafe() throws Exception { + String password = "password=stack-secret"; + String jdbcUrl = "jdbc:postgresql://private/database"; + String cliSecret = "--token=cli-secret"; IllegalStateException failure = new IllegalStateException("exception-secret"); + failure.setStackTrace(new StackTraceElement[] {new StackTraceElement( + password, jdbcUrl, cliSecret, "1.0", password, jdbcUrl, 17)}); + failure.addSuppressed(new IllegalArgumentException(cliSecret)); + failure.initCause(new IllegalArgumentException(jdbcUrl)); when(workflow.status()).thenThrow(failure); mvc.perform(get(SetupApiContract.STATUS_PATH).queryParam("token", "query-secret")) @@ -88,13 +100,27 @@ class SetupExceptionHandlerLoggingTest { ILoggingEvent event = errors.getFirst(); assertEquals("Unexpected setup request failure exception=java.lang.IllegalStateException", event.getFormattedMessage()); - assertNotNull(event.getThrowableProxy()); - assertEquals(Throwable.class.getName(), event.getThrowableProxy().getClassName()); - assertNull(event.getThrowableProxy().getMessage()); - assertNull(event.getThrowableProxy().getCause()); - assertTrue(event.getThrowableProxy().getStackTraceElementProxyArray().length > 0); - assertEquals(failure.getStackTrace()[0], - event.getThrowableProxy().getStackTraceElementProxyArray()[0].getStackTraceElement()); + assertNull(event.getThrowableProxy()); + String arguments = Arrays.toString(event.getArgumentArray()); + for (String secret : new String[] {password, jdbcUrl, cliSecret, "exception-secret"}) { + assertFalse(event.getFormattedMessage().contains(secret)); + assertFalse(arguments.contains(secret)); + } + } + + @Test + void diagnosticSinkFailureDoesNotChangeTheSafeHttpResponse() { + Clock clock = Clock.fixed(Instant.parse("2026-08-09T00:00:00Z"), ZoneOffset.UTC); + SetupExceptionHandler handler = new SetupExceptionHandler(clock, ignored -> { + throw new RuntimeException("diagnostic failure"); + }); + + var response = assertDoesNotThrow(() -> handler.unexpectedFailure(new IllegalStateException())); + + assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, response.getStatusCode()); + assertEquals("no-store", response.getHeaders().getFirst("Cache-Control")); + assertEquals(SetupErrorCode.INTERNAL_ERROR, response.getBody().errorCode()); + assertEquals(clock.instant(), response.getBody().observedAt()); } @Test diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/LoggingRecoveryFailureReporterTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/LoggingRecoveryFailureReporterTest.java index e5352b58c0..bccdd71db0 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/LoggingRecoveryFailureReporterTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/LoggingRecoveryFailureReporterTest.java @@ -19,29 +19,36 @@ package org.apache.hertzbeat.manager.setup.config; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.read.ListAppender; import java.io.IOException; +import java.util.Arrays; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; import org.slf4j.LoggerFactory; class LoggingRecoveryFailureReporterTest { @Test - void logsFixedStageStoreContextAndSafeDiagnosticThrowable() { + void logsOnlyFixedStageStoreAndExceptionClass() { Logger logger = (Logger) LoggerFactory.getLogger(LoggingRecoveryFailureReporter.class); ListAppender appender = new ListAppender<>(); appender.start(); logger.addAppender(appender); try { - IOException failure = new IOException("password=secret jdbc:postgresql://private/path"); - new LoggingRecoveryFailureReporter().report( + String password = "password=stack-secret"; + String jdbcUrl = "jdbc:postgresql://private/database"; + String cliSecret = "--token=cli-secret"; + IOException failure = new IOException("message-secret"); + failure.setStackTrace(new StackTraceElement[] {new StackTraceElement( + password, jdbcUrl, cliSecret, "1.0", password, jdbcUrl, 17)}); + failure.addSuppressed(new IOException(cliSecret)); + failure.initCause(new IOException(jdbcUrl)); + RecoveryFailureReporter.reportSafely(new LoggingRecoveryFailureReporter(), RecoveryFailureReporter.Stage.PROMOTE_CANDIDATE, RecoveryFailureReporter.Store.SECRET, failure); @@ -52,18 +59,36 @@ class LoggingRecoveryFailureReporterTest { assertEquals("Managed configuration recovery failure stage=PROMOTE_CANDIDATE store=SECRET " + "exception=java.io.IOException", event.getFormattedMessage()); - assertFalse(event.getFormattedMessage().contains("password")); - assertFalse(event.getFormattedMessage().contains("jdbc")); - assertNotNull(event.getThrowableProxy()); - assertEquals(Throwable.class.getName(), event.getThrowableProxy().getClassName()); - assertNull(event.getThrowableProxy().getMessage()); - assertNull(event.getThrowableProxy().getCause()); - assertTrue(event.getThrowableProxy().getStackTraceElementProxyArray().length > 0); - assertEquals(failure.getStackTrace()[0], - event.getThrowableProxy().getStackTraceElementProxyArray()[0].getStackTraceElement()); + assertNull(event.getThrowableProxy()); + String arguments = Arrays.toString(event.getArgumentArray()); + for (String secret : new String[] {password, jdbcUrl, cliSecret, "message-secret"}) { + assertFalse(event.getFormattedMessage().contains(secret)); + assertFalse(arguments.contains(secret)); + } } finally { logger.detachAppender(appender); appender.stop(); } } + + @Test + void customAdapterReceivesOnlyTheExceptionClass() { + AtomicReference capturedStage = new AtomicReference<>(); + AtomicReference capturedStore = new AtomicReference<>(); + AtomicReference capturedExceptionClass = new AtomicReference<>(); + RecoveryFailureReporter adapter = (stage, store, exceptionClass) -> { + capturedStage.set(stage); + capturedStore.set(store); + capturedExceptionClass.set(exceptionClass); + }; + + RecoveryFailureReporter.reportSafely(adapter, + RecoveryFailureReporter.Stage.RESTORE_ACTIVE, + RecoveryFailureReporter.Store.APPLICATION, + new IOException("raw failure must stay behind boundary")); + + assertEquals(RecoveryFailureReporter.Stage.RESTORE_ACTIVE, capturedStage.get()); + assertEquals(RecoveryFailureReporter.Store.APPLICATION, capturedStore.get()); + assertEquals(IOException.class.getName(), capturedExceptionClass.get()); + } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java index 054ef0af4b..6fda1c2ea6 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java @@ -21,10 +21,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.any; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.same; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -74,7 +72,25 @@ class ManagedConfigurationTransactionTest { transaction.apply(bundle("next"))); assertActivePair("previous"); verify(diagnostics).report(eq(RecoveryFailureReporter.Stage.PROMOTE_CANDIDATE), - eq(RecoveryFailureReporter.Store.SECRET), any(IOException.class)); + eq(RecoveryFailureReporter.Store.SECRET), eq(IOException.class.getName())); + } + + @Test + void reporterFailureDoesNotReplaceRollbackOutcome() throws Exception { + assertEquals(ManagedConfigurationTransaction.Outcome.APPLIED, + new ManagedConfigurationTransaction(installationRoot).apply(bundle("previous"))); + FileManagedApplicationConfigStore applicationStore = new FileManagedApplicationConfigStore(installationRoot); + FileManagedSecretStore failingSecrets = new FileManagedSecretStore( + installationRoot, new FailingOnceActivePublicationPublisher(new NioManagedFilePublisher())); + RecoveryFailureReporter failingReporter = (stage, store, exceptionClass) -> { + throw new RuntimeException("diagnostic failure"); + }; + ManagedConfigurationTransaction transaction = new ManagedConfigurationTransaction( + applicationStore, failingSecrets, installationRoot, failingReporter); + + assertEquals(ManagedConfigurationTransaction.Outcome.ROLLED_BACK, + transaction.apply(bundle("next"))); + assertActivePair("previous"); } @Test @@ -215,7 +231,27 @@ class ManagedConfigurationTransactionTest { applications, secretStore, installationRoot, diagnostics).recover()); verify(diagnostics).report(eq(RecoveryFailureReporter.Stage.DISCARD_CANDIDATE), - eq(RecoveryFailureReporter.Store.APPLICATION), same(failure)); + eq(RecoveryFailureReporter.Store.APPLICATION), eq(IOException.class.getName())); + } + + @Test + void reporterFailureDoesNotReplaceRecoveryRequiredOutcome() throws Exception { + ManagedApplicationConfigStore applications = mock(ManagedApplicationConfigStore.class); + ManagedSecretStore secretStore = mock(ManagedSecretStore.class); + when(applications.readActive()).thenReturn(CandidateRead.valid(configuration("owned"), "generation")); + when(applications.readCandidate()).thenReturn(CandidateRead.missing()); + when(applications.readLastKnownGood()).thenReturn(CandidateRead.missing()); + when(secretStore.readActive()).thenReturn(CandidateRead.valid(secrets("owned"), "generation")); + when(secretStore.readCandidate()).thenReturn(CandidateRead.missing()); + when(secretStore.readLastKnownGood()).thenReturn(CandidateRead.missing()); + doThrow(new IOException("discard failure")).when(applications).discardCandidate(); + RecoveryFailureReporter failingReporter = (stage, store, exceptionClass) -> { + throw new RuntimeException("diagnostic failure"); + }; + + assertEquals(ManagedConfigurationTransaction.Outcome.RECOVERY_REQUIRED, + new ManagedConfigurationTransaction( + applications, secretStore, installationRoot, failingReporter).recover()); } private static void waitForFile(Path ready) throws Exception { From 61f5dd325c34e84b73058a748f7c172f3a689335 Mon Sep 17 00:00:00 2001 From: Logic Date: Sun, 9 Aug 2026 06:12:23 +0800 Subject: [PATCH 22/71] Clear managed configuration buffers --- .../config/FileManagedSnapshotStore.java | 42 ++- .../setup/config/ManagedDocumentCodec.java | 2 + .../manager/setup/config/ManagedFileIo.java | 2 + .../FileManagedSnapshotStoreBufferTest.java | 306 ++++++++++++++++++ 4 files changed, 345 insertions(+), 7 deletions(-) create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/FileManagedSnapshotStoreBufferTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedSnapshotStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedSnapshotStore.java index c49c6e78ed..82f7fb10ef 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedSnapshotStore.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedSnapshotStore.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; +import java.util.Arrays; import java.util.Optional; final class FileManagedSnapshotStore { @@ -69,31 +70,35 @@ final class FileManagedSnapshotStore { void stageCandidate(T value, String generation) throws IOException { ensureSafePaths(); - publisher.publish(candidate, codec.encode(value, generation), ownerOnly); + publishEncoded(candidate, value, generation); } void promoteCandidate(T expected, String generation) throws IOException { ensureSafePaths(); + byte[] candidateDocument = null; ManagedDocumentCodec.Decoded decoded = null; CandidateRead activeRead = null; try { - byte[] candidateDocument = reader.read(candidate); + candidateDocument = reader.read(candidate); decoded = codec.decode(candidateDocument); if (!expected.equals(decoded.value()) || !generation.equals(decoded.generation())) { throw new IOException("Managed configuration candidate does not match the transaction"); } activeRead = readActive(); if (activeRead.state() == CandidateState.VALID) { - publisher.publish(lastKnownGood, codec.encode( - activeRead.value().orElseThrow(), activeRead.generation().orElseThrow()), ownerOnly); + publishEncoded(lastKnownGood, + activeRead.value().orElseThrow(), activeRead.generation().orElseThrow()); } else if (activeRead.state() != CandidateState.MISSING) { throw new IOException("Active managed configuration requires recovery"); } publisher.publish(active, candidateDocument, ownerOnly); + clear(candidateDocument); + candidateDocument = null; publisher.remove(candidate); } catch (ManagedDocumentCodec.DocumentException failure) { throw new IOException("A valid managed configuration candidate is required"); } finally { + clear(candidateDocument); close(decoded == null ? null : decoded.value()); if (activeRead != null) { activeRead.value().ifPresent(FileManagedSnapshotStore::close); @@ -105,8 +110,12 @@ final class FileManagedSnapshotStore { ensureSafePaths(); if (previous.isPresent()) { byte[] document = codec.encode(previous.orElseThrow(), generation.orElseThrow()); - publisher.publish(active, document, ownerOnly); - publisher.publish(lastKnownGood, document, ownerOnly); + try { + publisher.publish(active, document, ownerOnly); + publisher.publish(lastKnownGood, document, ownerOnly); + } finally { + clear(document); + } } else { publisher.remove(active); publisher.remove(lastKnownGood); @@ -122,8 +131,10 @@ final class FileManagedSnapshotStore { if (isUnsafePath(path)) { return CandidateRead.unreadable(); } + byte[] document = null; try { - ManagedDocumentCodec.Decoded decoded = codec.decode(reader.read(path)); + document = reader.read(path); + ManagedDocumentCodec.Decoded decoded = codec.decode(document); return CandidateRead.valid(decoded.value(), decoded.generation()); } catch (ManagedDocumentCodec.DocumentException exception) { return exception.state() == CandidateState.INVALID ? CandidateRead.invalid() : CandidateRead.corrupt(); @@ -131,6 +142,17 @@ final class FileManagedSnapshotStore { return CandidateRead.missing(); } catch (IOException exception) { return CandidateRead.unreadable(); + } finally { + clear(document); + } + } + + private void publishEncoded(Path target, T value, String generation) throws IOException { + byte[] document = codec.encode(value, generation); + try { + publisher.publish(target, document, ownerOnly); + } finally { + clear(document); } } @@ -158,4 +180,10 @@ final class FileManagedSnapshotStore { } } } + + private static void clear(byte[] content) { + if (content != null) { + Arrays.fill(content, (byte) 0); + } + } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedDocumentCodec.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedDocumentCodec.java index 72d8a66760..c6d531ba45 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedDocumentCodec.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedDocumentCodec.java @@ -27,8 +27,10 @@ import java.util.regex.Pattern; interface ManagedDocumentCodec { + /** Returns a caller-owned document buffer that the caller must clear after synchronous use. */ byte[] encode(T value, String generation); + /** Decodes synchronously and must not modify, retain, or asynchronously use the input buffer. */ Decoded decode(byte[] content) throws DocumentException; record Decoded(T value, String generation) { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedFileIo.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedFileIo.java index 86d553c47a..61c60281cc 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedFileIo.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedFileIo.java @@ -28,6 +28,7 @@ final class ManagedFileIo { interface Publisher { + /** Consumes synchronously and must not modify, retain, or asynchronously use the caller buffer. */ void publish(Path target, byte[] content, boolean ownerOnly) throws IOException; void remove(Path target) throws IOException; @@ -36,6 +37,7 @@ final class ManagedFileIo { @FunctionalInterface interface Reader { + /** Returns a caller-owned buffer to clear after its final synchronous use. */ byte[] read(Path path) throws IOException; } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/FileManagedSnapshotStoreBufferTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/FileManagedSnapshotStoreBufferTest.java new file mode 100644 index 0000000000..5e4c193e27 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/FileManagedSnapshotStoreBufferTest.java @@ -0,0 +1,306 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class FileManagedSnapshotStoreBufferTest { + + private static final String FILE_NAME = "managed-test.document"; + + @TempDir + private Path installationRoot; + + @Test + void stageClearsEncodedBuffersAfterSuccessfulAndFailedPublication() throws Exception { + CapturingCodec successCodec = new CapturingCodec(); + DiskPublisher successPublisher = new DiskPublisher(); + store(successCodec, successPublisher, path -> new byte[0]) + .stageCandidate("success", "generation-one"); + + assertAllZero(successCodec.encoded().getFirst()); + assertEquals("success|generation-one", Files.readString(candidatePath(), StandardCharsets.UTF_8)); + + CapturingCodec failureCodec = new CapturingCodec(); + DiskPublisher failurePublisher = new DiskPublisher(1); + assertThrows(IOException.class, () -> store(failureCodec, failurePublisher, path -> new byte[0]) + .stageCandidate("failure", "generation-two")); + + assertAllZero(failureCodec.encoded().getFirst()); + } + + @Test + void restoreKeepsOneBufferForBothPublishesAndClearsItAfterSuccessOrFailure() throws Exception { + CapturingCodec successCodec = new CapturingCodec(); + DiskPublisher successPublisher = new DiskPublisher(); + store(successCodec, successPublisher, path -> new byte[0]) + .restoreActive(Optional.of("previous"), Optional.of("generation-one")); + + byte[] successDocument = successCodec.encoded().getFirst(); + assertAllZero(successDocument); + assertEquals(List.of(activePath(), lastKnownGoodPath()), successPublisher.publishedTargets()); + assertEquals("previous|generation-one", Files.readString(activePath(), StandardCharsets.UTF_8)); + assertEquals("previous|generation-one", Files.readString(lastKnownGoodPath(), StandardCharsets.UTF_8)); + + CapturingCodec failureCodec = new CapturingCodec(); + DiskPublisher failurePublisher = new DiskPublisher(2); + assertThrows(IOException.class, () -> store(failureCodec, failurePublisher, path -> new byte[0]) + .restoreActive(Optional.of("failure"), Optional.of("generation-two"))); + + byte[] failedDocument = failureCodec.encoded().getFirst(); + assertAllZero(failedDocument); + assertEquals(List.of(activePath(), lastKnownGoodPath()), failurePublisher.publishedTargets()); + } + + @Test + void promotionClearsCandidateActiveAndLastKnownGoodBuffersAfterSuccess() throws Exception { + byte[] candidateDocument = document("next", "generation-two"); + byte[] activeDocument = document("previous", "generation-one"); + CapturingCodec codec = new CapturingCodec(); + DiskPublisher publisher = new DiskPublisher(); + FileManagedSnapshotStore store = store(codec, publisher, + path -> path.equals(candidatePath()) ? candidateDocument : activeDocument); + + store.promoteCandidate("next", "generation-two"); + + byte[] lastKnownGoodDocument = codec.encoded().getFirst(); + assertAllZero(candidateDocument); + assertAllZero(activeDocument); + assertAllZero(lastKnownGoodDocument); + assertEquals(List.of(lastKnownGoodPath(), activePath()), publisher.publishedTargets()); + assertEquals("next|generation-two", Files.readString(activePath(), StandardCharsets.UTF_8)); + assertEquals("previous|generation-one", Files.readString(lastKnownGoodPath(), StandardCharsets.UTF_8)); + } + + @Test + void promotionClearsEveryBufferWhenLastKnownGoodPublicationFails() { + byte[] candidateDocument = document("next", "generation-two"); + byte[] activeDocument = document("previous", "generation-one"); + CapturingCodec codec = new CapturingCodec(); + DiskPublisher publisher = new DiskPublisher(1); + FileManagedSnapshotStore store = store(codec, publisher, + path -> path.equals(candidatePath()) ? candidateDocument : activeDocument); + + assertThrows(IOException.class, () -> store.promoteCandidate("next", "generation-two")); + + assertAllZero(candidateDocument); + assertAllZero(activeDocument); + assertAllZero(codec.encoded().getFirst()); + } + + @Test + void promotionClearsEveryBufferWhenActivePublicationFails() { + byte[] candidateDocument = document("next", "generation-two"); + byte[] activeDocument = document("previous", "generation-one"); + CapturingCodec codec = new CapturingCodec(); + DiskPublisher publisher = new DiskPublisher(2); + FileManagedSnapshotStore store = store(codec, publisher, + path -> path.equals(candidatePath()) ? candidateDocument : activeDocument); + + assertThrows(IOException.class, () -> store.promoteCandidate("next", "generation-two")); + + assertAllZero(candidateDocument); + assertAllZero(activeDocument); + assertAllZero(codec.encoded().getFirst()); + assertEquals(List.of(lastKnownGoodPath(), activePath()), publisher.publishedTargets()); + } + + @Test + void promotionClearsCandidateBeforeEnteringSuccessfulRemove() throws Exception { + byte[] candidateDocument = document("next", "generation-two"); + AtomicBoolean removeObservedClearCandidate = new AtomicBoolean(); + DiskPublisher publisher = new DiskPublisher(-1, null, + ignored -> removeObservedClearCandidate.set(allZero(candidateDocument))); + FileManagedSnapshotStore store = store( + new CapturingCodec(), publisher, candidateOnlyReader(candidateDocument)); + + store.promoteCandidate("next", "generation-two"); + + assertTrue(removeObservedClearCandidate.get()); + assertAllZero(candidateDocument); + } + + @Test + void promotionClearsCandidateBeforeEnteringFailedRemove() { + byte[] candidateDocument = document("next", "generation-two"); + AtomicBoolean removeObservedClearCandidate = new AtomicBoolean(); + IOException removeFailure = new IOException("injected removal failure"); + DiskPublisher publisher = new DiskPublisher(-1, removeFailure, + ignored -> removeObservedClearCandidate.set(allZero(candidateDocument))); + FileManagedSnapshotStore store = store( + new CapturingCodec(), publisher, candidateOnlyReader(candidateDocument)); + + IOException thrown = assertThrows(IOException.class, + () -> store.promoteCandidate("next", "generation-two")); + + assertEquals(removeFailure, thrown); + assertTrue(removeObservedClearCandidate.get()); + assertAllZero(candidateDocument); + } + + @Test + void validationAndDecodeFailuresClearReaderOwnedBuffers() { + byte[] validationDocument = document("candidate", "generation-one"); + FileManagedSnapshotStore validatingStore = store( + new CapturingCodec(), new DiskPublisher(), path -> validationDocument); + + assertThrows(IOException.class, + () -> validatingStore.promoteCandidate("unexpected", "generation-one")); + assertAllZero(validationDocument); + + byte[] corruptDocument = document("corrupt", "generation-two"); + CapturingCodec corruptCodec = new CapturingCodec(true); + CandidateRead result = store(corruptCodec, new DiskPublisher(), path -> corruptDocument) + .readCandidate(); + + assertEquals(CandidateState.CORRUPT, result.state()); + assertAllZero(corruptDocument); + } + + private FileManagedSnapshotStore store( + ManagedDocumentCodec codec, + ManagedFileIo.Publisher publisher, + ManagedFileIo.Reader reader) { + return new FileManagedSnapshotStore<>( + installationRoot, FILE_NAME, false, codec, publisher, reader); + } + + private Path candidatePath() { + return installationRoot.resolve("data/config/" + FILE_NAME + ".candidate"); + } + + private Path activePath() { + return installationRoot.resolve("data/config/" + FILE_NAME); + } + + private Path lastKnownGoodPath() { + return installationRoot.resolve("data/config/" + FILE_NAME + ".last-known-good"); + } + + private ManagedFileIo.Reader candidateOnlyReader(byte[] candidateDocument) { + return path -> { + if (path.equals(candidatePath())) { + return candidateDocument; + } + throw new NoSuchFileException(path.toString()); + }; + } + + private static byte[] document(String value, String generation) { + return (value + "|" + generation).getBytes(StandardCharsets.UTF_8); + } + + private static void assertAllZero(byte[] content) { + assertTrue(content.length > 0); + assertArrayEquals(new byte[content.length], content); + } + + private static boolean allZero(byte[] content) { + for (byte value : content) { + if (value != 0) { + return false; + } + } + return true; + } + + private static final class CapturingCodec implements ManagedDocumentCodec { + private final List encoded = new ArrayList<>(); + private final boolean failDecode; + + private CapturingCodec() { + this(false); + } + + private CapturingCodec(boolean failDecode) { + this.failDecode = failDecode; + } + + @Override + public byte[] encode(String value, String generation) { + byte[] document = document(value, generation); + encoded.add(document); + return document; + } + + @Override + public Decoded decode(byte[] content) throws DocumentException { + if (failDecode) { + throw DocumentException.corrupt(); + } + String[] fields = new String(content, StandardCharsets.UTF_8).split("\\|", -1); + return new Decoded<>(fields[0], fields[1]); + } + + private List encoded() { + return encoded; + } + } + + private static final class DiskPublisher implements ManagedFileIo.Publisher { + private final List publishedTargets = new ArrayList<>(); + private final int failingPublish; + private final IOException removeFailure; + private final Consumer removeObserver; + + private DiskPublisher() { + this(-1, null, ignored -> { }); + } + + private DiskPublisher(int failingPublish) { + this(failingPublish, null, ignored -> { }); + } + + private DiskPublisher(int failingPublish, IOException removeFailure, Consumer removeObserver) { + this.failingPublish = failingPublish; + this.removeFailure = removeFailure; + this.removeObserver = removeObserver; + } + + @Override + public void publish(Path target, byte[] content, boolean ownerOnly) throws IOException { + assertFalse(allZero(content)); + publishedTargets.add(target); + if (publishedTargets.size() == failingPublish) { + throw new IOException("injected publication failure"); + } + Files.createDirectories(target.getParent()); + Files.write(target, content); + } + + @Override + public void remove(Path target) throws IOException { + removeObserver.accept(target); + if (removeFailure != null) { + throw removeFailure; + } + Files.deleteIfExists(target); + } + + private List publishedTargets() { + return publishedTargets; + } + } +} From 6dd503e83b7f912217fc30b180b3b5c189d1f1d5 Mon Sep 17 00:00:00 2001 From: Logic Date: Sun, 9 Aug 2026 06:42:06 +0800 Subject: [PATCH 23/71] Centralize optional setup transitions --- .../setup/api/SetupApiConfiguration.java | 16 +- .../setup/workflow/DefaultSetupWorkflow.java | 39 +---- .../OptionalConfigurationProjection.java | 40 +++++ .../workflow/SetupTransitionService.java | 23 +++ .../workflow/DefaultSetupWorkflowTest.java | 7 +- .../HeadlessSetupCoordinatorTest.java | 1 + .../workflow/SetupTransitionServiceTest.java | 139 +++++++++++++++++- 7 files changed, 219 insertions(+), 46 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/OptionalConfigurationProjection.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java index 4b5cf86e92..b46b145260 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java @@ -133,25 +133,31 @@ public class SetupApiConfiguration { return new SetupMutationSerializer(); } + @Bean + public SetupOptionsCoordinator setupOptionsCoordinator(Environment environment) { + return new SetupOptionsCoordinator(new ManagedConfigurationTransaction( + SetupInstallationPaths.root(environment))); + } + @Bean public SetupTransitionService setupTransitionService( SetupRuntimeState state, SetupRequestValidator validator, SetupConfigurationCoordinator configuration, ManagedConfigCapability capability, + SetupOptionsCoordinator options, ObjectProvider identityProvider, ObjectProvider installationProvider, Environment environment) { - return new SetupTransitionService(state, validator, configuration, capability, + return new SetupTransitionService(state, validator, configuration, capability, options, identityProvider.stream().findFirst(), completion(environment, installationProvider.stream().findFirst())); } @Bean - public DefaultSetupWorkflow setupWorkflow(Environment environment, SetupRuntimeState state, + public DefaultSetupWorkflow setupWorkflow(SetupRuntimeState state, SetupRequestValidator validator, SetupOperationRegistry operations, SetupMutationSerializer mutations, SetupTransitionService transitions) { - return new DefaultSetupWorkflow(state, validator, operations, - new SetupOptionsCoordinator(new ManagedConfigurationTransaction( - SetupInstallationPaths.root(environment))), Clock.systemUTC(), mutations, transitions); + return new DefaultSetupWorkflow( + state, validator, operations, Clock.systemUTC(), mutations, transitions); } @Bean diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflow.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflow.java index 0f9a237f79..93b6cf8174 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflow.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflow.java @@ -27,7 +27,6 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationResp import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportRequest; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OperationResponse; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionalConfigurationSummary; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsRequest; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; @@ -37,10 +36,8 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.UnlockRequest; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.UnlockResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidateRequest; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationResponse; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationSection; import org.apache.hertzbeat.manager.setup.api.SetupApiException; import org.apache.hertzbeat.manager.setup.api.SetupWorkflow; -import org.apache.hertzbeat.manager.setup.config.ManagedOptionalConfiguration.ServerInstrumentationSettings; import org.springframework.http.HttpStatus; /** Cohesive setup state-machine facade; transport and persistence remain in dedicated collaborators. */ @@ -48,19 +45,17 @@ public final class DefaultSetupWorkflow implements SetupWorkflow { private final SetupRuntimeState state; private final SetupRequestValidator validator; private final SetupOperationRegistry operations; - private final SetupOptionsCoordinator options; private final Clock clock; private final SetupMutationSerializer mutations; private final SetupTransitionService transitions; public DefaultSetupWorkflow(SetupRuntimeState state, SetupRequestValidator validator, SetupOperationRegistry operations, - SetupOptionsCoordinator options, Clock clock, SetupMutationSerializer mutations, + Clock clock, SetupMutationSerializer mutations, SetupTransitionService transitions) { this.state = state; this.validator = validator; this.operations = operations; - this.options = options; this.clock = clock; this.mutations = mutations; this.transitions = transitions; @@ -114,30 +109,7 @@ public final class DefaultSetupWorkflow implements SetupWorkflow { } private OptionsResponse configureOptionsMutation(OptionsRequest request) { - requireWritable(); - state.ensurePhase(SetupPhase.OPTIONAL_CONFIGURATION); - if (request.serverInstrumentation() != null) { - requireValid(new ValidateRequest(ValidationSection.SERVER_INSTRUMENTATION, - null, null, request.serverInstrumentation(), null)); - } - if (request.mail() != null) { - requireValid(new ValidateRequest(ValidationSection.MAIL, - null, null, null, request.mail())); - } - options.persist(request); - OptionalConfigurationSummary summary = new OptionalConfigurationSummary( - request.serverInstrumentation() != null - && ServerInstrumentationSettings.normalize( - request.serverInstrumentation().serverOtlpHttpEndpoint()).isPresent(), - request.serverInstrumentation() != null - && ServerInstrumentationSettings.normalize( - request.serverInstrumentation().serverOtlpGrpcEndpoint()).isPresent(), - request.retention() != null, request.mail() != null); - state.optionsConfigured(summary, - SetupWarningPolicy.INSTANCE.evaluate(state.managementDatabaseKind(), request)); - return new OptionsResponse(summary.serverOtlpHttpConfigured(), summary.serverOtlpGrpcConfigured(), - summary.retentionConfigured(), summary.mailConfigured(), - SetupPhase.OPTIONAL_CONFIGURATION); + return transitions.configureOptions(request); } @Override @@ -166,11 +138,4 @@ public final class DefaultSetupWorkflow implements SetupWorkflow { } } - private void requireValid(ValidateRequest request) { - ValidationResponse response = validator.validate(request); - if (!response.valid()) { - throw new SetupApiException(response.errorCode(), HttpStatus.BAD_REQUEST); - } - } - } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/OptionalConfigurationProjection.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/OptionalConfigurationProjection.java new file mode 100644 index 0000000000..00e59d8497 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/OptionalConfigurationProjection.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.List; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionalConfigurationSummary; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; +import org.apache.hertzbeat.manager.setup.config.ManagedOptionalConfiguration.ServerInstrumentationSettings; + +/** Projects persisted optional settings into the secret-free runtime and response shape. */ +record OptionalConfigurationProjection( + OptionalConfigurationSummary summary, List warnings) { + + static OptionalConfigurationProjection from(MetadataDatabaseKind databaseKind, OptionsRequest request) { + OptionalConfigurationSummary summary = new OptionalConfigurationSummary( + request.serverInstrumentation() != null + && ServerInstrumentationSettings.normalize( + request.serverInstrumentation().serverOtlpHttpEndpoint()).isPresent(), + request.serverInstrumentation() != null + && ServerInstrumentationSettings.normalize( + request.serverInstrumentation().serverOtlpGrpcEndpoint()).isPresent(), + request.retention() != null, request.mail() != null); + return new OptionalConfigurationProjection( + summary, SetupWarningPolicy.INSTANCE.evaluate(databaseKind, request)); + } + + OptionsResponse response() { + return new OptionsResponse(summary.serverOtlpHttpConfigured(), summary.serverOtlpGrpcConfigured(), + summary.retentionConfigured(), summary.mailConfigured(), SetupPhase.OPTIONAL_CONFIGURATION); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionService.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionService.java index 83eebf013b..24828705b3 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionService.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionService.java @@ -14,6 +14,8 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.AdministratorRequ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.CompleteRequest; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationRequest; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; @@ -34,17 +36,20 @@ public final class SetupTransitionService { private final SetupRequestValidator validator; private final SetupConfigurationCoordinator configuration; private final ManagedConfigCapability capability; + private final SetupOptionsCoordinator options; private final Optional identities; private final Optional completion; public SetupTransitionService(SetupRuntimeState state, SetupRequestValidator validator, SetupConfigurationCoordinator configuration, ManagedConfigCapability capability, + SetupOptionsCoordinator options, Optional identities, Optional completion) { this.state = state; this.validator = validator; this.configuration = configuration; this.capability = capability; + this.options = options; this.identities = identities; this.completion = completion; } @@ -83,6 +88,24 @@ public final class SetupTransitionService { } } + public OptionsResponse configureOptions(OptionsRequest request) { + requireWritable(); + state.ensurePhase(SetupPhase.OPTIONAL_CONFIGURATION); + if (request.serverInstrumentation() != null) { + requireValid(validator, new ValidateRequest(ValidationSection.SERVER_INSTRUMENTATION, + null, null, request.serverInstrumentation(), null)); + } + if (request.mail() != null) { + requireValid(validator, new ValidateRequest(ValidationSection.MAIL, + null, null, null, request.mail())); + } + options.persist(request); + OptionalConfigurationProjection projection = OptionalConfigurationProjection.from( + state.managementDatabaseKind(), request); + state.optionsConfigured(projection.summary(), projection.warnings()); + return projection.response(); + } + public String complete(CompletionCommand command) { requireWritable(); state.ensurePhase(SetupPhase.OPTIONAL_CONFIGURATION); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java index 8cf8129d6a..52f2badf3a 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java @@ -240,9 +240,9 @@ class DefaultSetupWorkflowTest { Optional completion, SetupOptionsCoordinator options, Clock clock, SetupMutationSerializer mutations) { SetupTransitionService transitions = new SetupTransitionService( - state, validator, configuration, capability, identities, completion); + state, validator, configuration, capability, options, identities, completion); return new DefaultSetupWorkflow(state, validator, operations, - options, clock, mutations, transitions); + clock, mutations, transitions); } private static HeadlessSetupCoordinator headless( @@ -251,7 +251,8 @@ class DefaultSetupWorkflowTest { Optional identities, Optional completion, SetupMutationSerializer mutations) { SetupTransitionService transitions = new SetupTransitionService( - state, validator, configuration, capability, identities, completion); + state, validator, configuration, capability, + mock(SetupOptionsCoordinator.class), identities, completion); return new HeadlessSetupCoordinator(state, mutations, transitions); } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinatorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinatorTest.java index aecb2135c6..921622537d 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinatorTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinatorTest.java @@ -46,6 +46,7 @@ class HeadlessSetupCoordinatorTest { SetupCompletionCoordinator completion = mock(SetupCompletionCoordinator.class); SetupTransitionService transitions = new SetupTransitionService(state, mock(SetupRequestValidator.class), mock(SetupConfigurationCoordinator.class), capability, + mock(SetupOptionsCoordinator.class), Optional.of(mock(IdentityInitializationService.class)), Optional.of(completion)); HeadlessSetupCoordinator coordinator = new HeadlessSetupCoordinator( state, new SetupMutationSerializer(), transitions); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionServiceTest.java index f9f47b7240..f142bd631e 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionServiceTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionServiceTest.java @@ -10,6 +10,7 @@ package org.apache.hertzbeat.manager.setup.workflow; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.doThrow; @@ -25,16 +26,27 @@ import java.util.Optional; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.AdministratorRequest; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.CompleteRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigSource; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationRequest; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ManagementDatabaseSummary; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionalConfigurationSummary; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ServerInstrumentationConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupAccess; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreSummary; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidateRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationSection; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiException; import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; @@ -197,6 +209,92 @@ class SetupTransitionServiceTest { verifyNoInteractions(validator, configuration); } + @Test + void optionsRejectWrongPhaseBeforeValidationOrPersistence() { + ManagedConfigCapability capability = mock(ManagedConfigCapability.class); + SetupRequestValidator validator = mock(SetupRequestValidator.class); + SetupOptionsCoordinator options = mock(SetupOptionsCoordinator.class); + SetupTransitionService transitions = transitions( + state(capability, SetupPhase.ADMINISTRATOR_REQUIRED, false, null), validator, + mock(SetupConfigurationCoordinator.class), capability, options, + Optional.empty(), Optional.empty()); + + assertThatThrownBy(() -> transitions.configureOptions(optionsRequest())) + .isInstanceOf(SetupWorkflowConflict.class); + + verifyNoInteractions(validator, options); + } + + @Test + void serverInstrumentationValidationFailureDoesNotPersistOrPublishState() { + assertOptionsValidationFailure(new OptionsRequest( + new ServerInstrumentationConfiguration("not-an-endpoint", null), null, null), + ValidationSection.SERVER_INSTRUMENTATION, SetupErrorCode.SERVER_INSTRUMENTATION_INVALID); + } + + @Test + void mailValidationFailureDoesNotPersistOrPublishState() { + assertOptionsValidationFailure(new OptionsRequest(null, null, + new MailConfiguration("mail.example.test", 25, MailSecurity.STARTTLS, + null, null, "alerts@example.test")), + ValidationSection.MAIL, SetupErrorCode.MAIL_CONNECTION_FAILED); + } + + @Test + void optionsPersistenceFailureDoesNotPublishSummaryOrWarnings() { + ManagedConfigCapability capability = mock(ManagedConfigCapability.class); + SetupRuntimeState state = state(capability, SetupPhase.OPTIONAL_CONFIGURATION, true, "operator"); + SetupRequestValidator validator = mock(SetupRequestValidator.class); + when(validator.validate(any(ValidateRequest.class))) + .thenReturn(new ValidationResponse(true, CLOCK.instant(), null, List.of())); + SetupOptionsCoordinator options = mock(SetupOptionsCoordinator.class); + doThrow(new SetupApiException(SetupErrorCode.CONFIG_WRITE_FAILED, + org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR)).when(options).persist(any()); + SetupTransitionService transitions = transitions(state, validator, + mock(SetupConfigurationCoordinator.class), capability, options, + Optional.empty(), Optional.empty()); + var before = state.status(); + + assertThatThrownBy(() -> transitions.configureOptions(optionsRequest())) + .isInstanceOf(SetupApiException.class); + + assertThat(state.status().optional()).isEqualTo(before.optional()); + assertThat(state.pendingWarnings()).isEqualTo(before.pendingWarnings()); + } + + @Test + void successfulOptionsNormalizeSummaryAndUseCurrentManagementDatabaseWarningPolicy() { + ManagedConfigCapability capability = mock(ManagedConfigCapability.class); + SetupConfigurationProjection projection = new SetupConfigurationProjection( + new ManagementDatabaseSummary(MetadataDatabaseKind.POSTGRESQL, true, + ConfigSource.UI_MANAGED, false), + new TelemetryStoreSummary(TelemetryStoreKind.GREPTIME, true, + ConfigSource.UI_MANAGED, false), + new OptionalConfigurationSummary(false, false, false, false), List.of()); + SetupRuntimeState state = new SetupRuntimeState(CLOCK, capability, + SetupPhase.OPTIONAL_CONFIGURATION, SetupAccess.LOCAL, true, "operator", projection); + SetupRequestValidator validator = mock(SetupRequestValidator.class); + when(validator.validate(any(ValidateRequest.class))) + .thenReturn(new ValidationResponse(true, CLOCK.instant(), null, List.of())); + SetupOptionsCoordinator options = mock(SetupOptionsCoordinator.class); + SetupTransitionService transitions = transitions(state, validator, + mock(SetupConfigurationCoordinator.class), capability, options, + Optional.empty(), Optional.empty()); + OptionsRequest request = new OptionsRequest( + new ServerInstrumentationConfiguration(" ", "https://server.example.test:4317"), null, + new MailConfiguration("mail.example.test", 25, MailSecurity.NONE, + null, null, "alerts@example.test")); + + var response = transitions.configureOptions(request); + + assertThat(response.serverOtlpHttpConfigured()).isFalse(); + assertThat(response.serverOtlpGrpcConfigured()).isTrue(); + assertThat(state.status().optional()).isEqualTo( + new OptionalConfigurationSummary(false, true, false, true)); + assertThat(state.pendingWarnings()).containsExactly(SetupWarningCode.MAIL_SECURITY_NONE); + verify(options).persist(request); + } + @Test void browserAndHeadlessCompletionUseTheSameWarningGateAndCompletionSideEffect() { ManagedConfigCapability capability = mock(ManagedConfigCapability.class); @@ -223,7 +321,41 @@ class SetupTransitionServiceTest { SetupRuntimeState state, SetupRequestValidator validator, SetupConfigurationCoordinator configuration, ManagedConfigCapability capability, Optional identities, Optional completion) { - return new SetupTransitionService(state, validator, configuration, capability, identities, completion); + return transitions(state, validator, configuration, capability, + mock(SetupOptionsCoordinator.class), identities, completion); + } + + private static SetupTransitionService transitions( + SetupRuntimeState state, SetupRequestValidator validator, + SetupConfigurationCoordinator configuration, ManagedConfigCapability capability, + SetupOptionsCoordinator options, Optional identities, + Optional completion) { + return new SetupTransitionService( + state, validator, configuration, capability, options, identities, completion); + } + + private static void assertOptionsValidationFailure( + OptionsRequest request, ValidationSection section, SetupErrorCode errorCode) { + ManagedConfigCapability capability = mock(ManagedConfigCapability.class); + SetupRuntimeState state = state(capability, SetupPhase.OPTIONAL_CONFIGURATION, true, "operator"); + SetupRequestValidator validator = mock(SetupRequestValidator.class); + when(validator.validate(any(ValidateRequest.class))) + .thenReturn(new ValidationResponse(false, CLOCK.instant(), errorCode, List.of())); + SetupOptionsCoordinator options = mock(SetupOptionsCoordinator.class); + SetupTransitionService transitions = transitions(state, validator, + mock(SetupConfigurationCoordinator.class), capability, options, + Optional.empty(), Optional.empty()); + var before = state.status(); + + assertThatThrownBy(() -> transitions.configureOptions(request)) + .isInstanceOfSatisfying(SetupApiException.class, + failure -> assertThat(failure.errorCode()).isEqualTo(errorCode)); + + verifyNoInteractions(options); + verify(validator).validate(argThat( + (ValidateRequest validation) -> validation.section() == section)); + assertThat(state.status().optional()).isEqualTo(before.optional()); + assertThat(state.pendingWarnings()).isEqualTo(before.pendingWarnings()); } private static SetupRuntimeState state(ManagedConfigCapability capability, SetupPhase phase, @@ -248,6 +380,11 @@ class SetupTransitionServiceTest { "localhost:4001", "http://localhost:4000", "public", null, null)); } + private static OptionsRequest optionsRequest() { + return new OptionsRequest( + new ServerInstrumentationConfiguration("https://server.example.test:4318", null), null, null); + } + private static HeadlessSetupWorkflow.RequiredConfiguration headlessConfiguration(SecretValue password) { return headlessConfiguration(SetupPhase.CONFIGURATION_REQUIRED, ApplyMode.MANAGED_WRITE, password); } From e6a45c721c8f3320c5638e9538cd50a8b3a9ff00 Mon Sep 17 00:00:00 2001 From: Logic Date: Sun, 9 Aug 2026 06:57:54 +0800 Subject: [PATCH 24/71] Unify setup configuration orchestration --- .../SetupConfigurationCoordinator.java | 53 ++--- .../workflow/SetupTransitionService.java | 2 +- .../workflow/DefaultSetupWorkflowTest.java | 3 +- .../SetupConfigurationCoordinatorTest.java | 208 +++++++++++++++--- .../workflow/SetupTransitionServiceTest.java | 6 +- 5 files changed, 204 insertions(+), 68 deletions(-) diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationCoordinator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationCoordinator.java index 1ca187b151..5bbdaa60cb 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationCoordinator.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationCoordinator.java @@ -42,17 +42,33 @@ public final class SetupConfigurationCoordinator { } public ConfigurationResponse configure(ConfigurationRequest request, ManagedConfigCapability capability) { - if (!configurationPhase(request.expectedPhase()) || request.applyMode() != capability.applyMode()) { + try (ManagedConfigurationBundle bundle = SetupConfigurationMapper.map(request)) { + return configureMapped(request.expectedPhase(), request.applyMode(), bundle, capability); + } + } + + public ConfigurationResponse configure(HeadlessSetupWorkflow.RequiredConfiguration request, + ManagedConfigCapability capability) { + // Mapping copies caller-owned secrets; this scope owns and clears only the copy. + try (ManagedConfigurationBundle bundle = SetupConfigurationMapper.map(request)) { + return configureMapped(request.expectedPhase(), request.applyMode(), bundle, capability); + } + } + + private ConfigurationResponse configureMapped( + SetupPhase expectedPhase, ApplyMode applyMode, + ManagedConfigurationBundle bundle, ManagedConfigCapability capability) { + if (!configurationPhase(expectedPhase) || applyMode != capability.applyMode()) { throw new SetupWorkflowConflict(); } - String operationId = beginOperation(request.expectedPhase()); - if (request.applyMode() == ApplyMode.EXTERNAL_APPLY) { + String operationId = beginOperation(expectedPhase); + if (applyMode == ApplyMode.EXTERNAL_APPLY) { operations.finish(operationId, SetupOperationState.AWAITING_EXTERNAL_APPLY, SetupPhase.EXTERNAL_APPLY_REQUIRED, null, true); return response(operationId); } try { - return applyManaged(operationId, request); + return applyManaged(operationId, bundle); } catch (IOException failure) { operations.finish(operationId, SetupOperationState.FAILED, SetupPhase.CONFIGURATION_REQUIRED, SetupErrorCode.CONFIG_WRITE_FAILED, false); @@ -60,35 +76,6 @@ public final class SetupConfigurationCoordinator { } } - public ConfigurationResponse configure(HeadlessSetupWorkflow.RequiredConfiguration request, - ManagedConfigurationBundle bundle, - ManagedConfigCapability capability) { - try (bundle) { - if (!configurationPhase(request.expectedPhase()) || request.applyMode() != capability.applyMode()) { - throw new SetupWorkflowConflict(); - } - String operationId = beginOperation(request.expectedPhase()); - if (request.applyMode() == ApplyMode.EXTERNAL_APPLY) { - operations.finish(operationId, SetupOperationState.AWAITING_EXTERNAL_APPLY, - SetupPhase.EXTERNAL_APPLY_REQUIRED, null, true); - return response(operationId); - } - try { - return applyManaged(operationId, bundle); - } catch (IOException failure) { - operations.finish(operationId, SetupOperationState.FAILED, - SetupPhase.CONFIGURATION_REQUIRED, SetupErrorCode.CONFIG_WRITE_FAILED, false); - throw new SetupApiException(SetupErrorCode.CONFIG_WRITE_FAILED, HttpStatus.INTERNAL_SERVER_ERROR); - } - } - } - - private ConfigurationResponse applyManaged(String operationId, ConfigurationRequest request) throws IOException { - try (ManagedConfigurationBundle bundle = SetupConfigurationMapper.map(request)) { - return applyManaged(operationId, bundle); - } - } - private ConfigurationResponse applyManaged(String operationId, ManagedConfigurationBundle bundle) throws IOException { ManagedConfigurationTransaction.Outcome outcome = transaction.apply(bundle); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionService.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionService.java index 24828705b3..6990c2fb83 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionService.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionService.java @@ -212,7 +212,7 @@ public final class SetupTransitionService { @Override public ConfigurationResponse configure(SetupConfigurationCoordinator coordinator, ManagedConfigCapability capability) { - return coordinator.configure(request, SetupConfigurationMapper.map(request), capability); + return coordinator.configure(request, capability); } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java index 52f2badf3a..8ed8cba14e 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java @@ -57,7 +57,6 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationRespons import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidateRequest; import org.apache.hertzbeat.manager.setup.api.SetupApiException; import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; -import org.apache.hertzbeat.manager.setup.config.ManagedConfigurationBundle; import org.apache.hertzbeat.manager.setup.config.SecretValue; import org.apache.hertzbeat.manager.setup.identity.IdentityInitializationService; import org.junit.jupiter.api.Test; @@ -230,7 +229,7 @@ class DefaultSetupWorkflowTest { .isInstanceOf(SetupWorkflowConflict.class); } verify(configuration, never()).configure(any(HeadlessSetupWorkflow.RequiredConfiguration.class), - any(ManagedConfigurationBundle.class), eq(capability)); + eq(capability)); } private static DefaultSetupWorkflow workflow( diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationCoordinatorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationCoordinatorTest.java index 69fbf9e926..ed4be254d3 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationCoordinatorTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationCoordinatorTest.java @@ -23,15 +23,29 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; +import java.io.IOException; import java.nio.file.Path; import java.time.Clock; import java.util.Arrays; import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.manager.setup.api.SetupApiException; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationRequest; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OperationResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreConfiguration; @@ -39,6 +53,7 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreKin import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector; import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; import org.apache.hertzbeat.manager.setup.config.ManagedConfigDeploymentDetector; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigurationBundle; import org.apache.hertzbeat.manager.setup.config.ManagedConfigurationTransaction; import org.apache.hertzbeat.manager.setup.config.DeploymentConstraint; import org.apache.hertzbeat.manager.setup.config.SecretValue; @@ -88,45 +103,105 @@ class SetupConfigurationCoordinatorTest { @Test void externalApplyReentryReturnsFreshAcknowledgementWithoutPersistingSubmittedSecrets() { - SetupOperationRegistry operations = new SetupOperationRegistry(Clock.systemUTC()); - SetupConfigurationCoordinator coordinator = new SetupConfigurationCoordinator( - new ManagedConfigurationTransaction(installationRoot), operations); ManagedConfigCapability capability = new ManagedConfigCapability( ApplyMode.EXTERNAL_APPLY, false, DeploymentConstraint.READ_ONLY); - var first = coordinator.configure(request(SetupPhase.CONFIGURATION_REQUIRED, ApplyMode.EXTERNAL_APPLY), - capability); - var replacement = coordinator.configure( - request(SetupPhase.EXTERNAL_APPLY_REQUIRED, ApplyMode.EXTERNAL_APPLY), capability); + for (Transport transport : Transport.values()) { + ManagedConfigurationTransaction transaction = mock(ManagedConfigurationTransaction.class); + SetupOperationRegistry operations = new SetupOperationRegistry(Clock.systemUTC()); + SetupConfigurationCoordinator coordinator = new SetupConfigurationCoordinator(transaction, operations); - assertNotEquals(first.operationId(), replacement.operationId()); - assertEquals(SetupOperationState.AWAITING_EXTERNAL_APPLY, replacement.state()); - assertEquals(SetupPhase.EXTERNAL_APPLY_REQUIRED, replacement.phase()); - assertTrue(replacement.exportAvailable()); - assertEquals(ManagedActiveConfigurationInspector.State.ABSENT, - new ManagedActiveConfigurationInspector(installationRoot).inspect().state()); + var first = configure(transport, coordinator, SetupPhase.CONFIGURATION_REQUIRED, + ApplyMode.EXTERNAL_APPLY, capability); + var replacement = configure(transport, coordinator, SetupPhase.EXTERNAL_APPLY_REQUIRED, + ApplyMode.EXTERNAL_APPLY, capability); + + assertNotEquals(first.operationId(), replacement.operationId(), transport.name()); + assertEquals(SetupOperationState.AWAITING_EXTERNAL_APPLY, replacement.state(), transport.name()); + assertEquals(SetupPhase.EXTERNAL_APPLY_REQUIRED, replacement.phase(), transport.name()); + assertTrue(replacement.exportAvailable(), transport.name()); + assertEquals(SetupOperationState.AWAITING_EXTERNAL_APPLY, + operations.get(first.operationId()).state(), transport.name()); + verifyNoInteractions(transaction); + } } @Test - void headlessExternalApplyClosesTheCoordinatorOwnedSecretBundle() { - SetupConfigurationCoordinator coordinator = new SetupConfigurationCoordinator( - new ManagedConfigurationTransaction(installationRoot), - new SetupOperationRegistry(Clock.systemUTC())); - try (SecretValue callerPassword = SecretValue.of("metadata-password")) { - var request = new HeadlessSetupWorkflow.RequiredConfiguration(SetupPhase.CONFIGURATION_REQUIRED, - ApplyMode.EXTERNAL_APPLY, - new HeadlessSetupWorkflow.Metadata(MetadataDatabaseKind.H2, - "jdbc:h2:./data/setup", "sa", callerPassword), - new HeadlessSetupWorkflow.Telemetry("localhost:4001", "http://localhost:4000", - "public", Optional.empty(), Optional.empty())); - var bundle = SetupConfigurationMapper.map(request); + void bothTransportsRejectPhaseAndApplyModeMismatchBeforeOperationOrTransaction() { + for (Transport transport : Transport.values()) { + assertRejectedBeforeOperation(transport, SetupPhase.ADMINISTRATOR_REQUIRED, + ApplyMode.MANAGED_WRITE, managedCapability()); + assertRejectedBeforeOperation(transport, SetupPhase.CONFIGURATION_REQUIRED, + ApplyMode.MANAGED_WRITE, externalCapability()); + } + } - coordinator.configure(request, bundle, - new ManagedConfigCapability(ApplyMode.EXTERNAL_APPLY, false, - DeploymentConstraint.READ_ONLY)); + @Test + void bothTransportsPublishManagedAppliedAsAwaitingRestart() throws IOException { + for (Transport transport : Transport.values()) { + ManagedConfigurationTransaction transaction = mock(ManagedConfigurationTransaction.class); + when(transaction.apply(any())).thenReturn(ManagedConfigurationTransaction.Outcome.APPLIED); + SetupConfigurationCoordinator coordinator = new SetupConfigurationCoordinator( + transaction, new SetupOperationRegistry(Clock.systemUTC())); + + var response = configure(transport, coordinator, SetupPhase.CONFIGURATION_REQUIRED, + ApplyMode.MANAGED_WRITE, managedCapability()); + + assertEquals(SetupOperationState.AWAITING_RESTART, response.state(), transport.name()); + assertEquals(SetupPhase.APPLICATION_STARTING, response.phase(), transport.name()); + assertFalse(response.exportAvailable(), transport.name()); + verify(transaction).apply(any()); + } + } + + @Test + void bothTransportsMapTransactionOutcomesToStableOperationFailures() throws IOException { + for (Transport transport : Transport.values()) { + assertTransactionFailure(transport, ManagedConfigurationTransaction.Outcome.ROLLED_BACK, + SetupOperationState.ROLLED_BACK, SetupPhase.CONFIGURATION_REQUIRED, + SetupErrorCode.CONFIG_WRITE_FAILED); + assertTransactionFailure(transport, ManagedConfigurationTransaction.Outcome.RECOVERY_REQUIRED, + SetupOperationState.FAILED, SetupPhase.RECOVERY_REQUIRED, + SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + } + + @Test + void bothTransportsMapIoFailureToStableFailedOperation() throws IOException { + for (Transport transport : Transport.values()) { + ManagedConfigurationTransaction transaction = mock(ManagedConfigurationTransaction.class); + doThrow(new IOException("controlled failure")).when(transaction).apply(any()); + SetupOperationRegistry operations = spy(new SetupOperationRegistry(Clock.systemUTC())); + SetupConfigurationCoordinator coordinator = new SetupConfigurationCoordinator(transaction, operations); + + SetupApiException failure = assertThrows(SetupApiException.class, + () -> configure(transport, coordinator, SetupPhase.CONFIGURATION_REQUIRED, + ApplyMode.MANAGED_WRITE, managedCapability())); + + assertEquals(SetupErrorCode.CONFIG_WRITE_FAILED, failure.errorCode(), transport.name()); + assertFinishedOperation(operations, SetupOperationState.FAILED, + SetupPhase.CONFIGURATION_REQUIRED, SetupErrorCode.CONFIG_WRITE_FAILED); + } + } + + @Test + void headlessConfigureClosesItsCopiedBundleWithoutClosingCallerSecret() throws IOException { + ManagedConfigurationTransaction transaction = mock(ManagedConfigurationTransaction.class); + AtomicReference applied = new AtomicReference<>(); + when(transaction.apply(any())).thenAnswer(invocation -> { + applied.set(invocation.getArgument(0)); + return ManagedConfigurationTransaction.Outcome.APPLIED; + }); + SetupConfigurationCoordinator coordinator = new SetupConfigurationCoordinator( + transaction, new SetupOperationRegistry(Clock.systemUTC())); + try (SecretValue callerPassword = SecretValue.of("metadata-password")) { + var request = headlessRequest(SetupPhase.CONFIGURATION_REQUIRED, ApplyMode.MANAGED_WRITE, + callerPassword); + + coordinator.configure(request, managedCapability()); assertArrayEquals(new char["metadata-password".length()], - bundle.secrets().metadataDatabasePassword().copy()); + applied.get().secrets().metadataDatabasePassword().copy()); char[] retained = callerPassword.copy(); try { assertArrayEquals("metadata-password".toCharArray(), retained); @@ -136,6 +211,81 @@ class SetupConfigurationCoordinatorTest { } } + private static void assertRejectedBeforeOperation( + Transport transport, SetupPhase phase, ApplyMode applyMode, ManagedConfigCapability capability) { + ManagedConfigurationTransaction transaction = mock(ManagedConfigurationTransaction.class); + SetupOperationRegistry operations = spy(new SetupOperationRegistry(Clock.systemUTC())); + SetupConfigurationCoordinator coordinator = new SetupConfigurationCoordinator(transaction, operations); + + assertThrows(SetupWorkflowConflict.class, + () -> configure(transport, coordinator, phase, applyMode, capability), transport.name()); + + verify(operations, never()).begin(any()); + verify(operations, never()).replaceExternalApply(any()); + verifyNoInteractions(transaction); + } + + private static void assertTransactionFailure( + Transport transport, ManagedConfigurationTransaction.Outcome outcome, + SetupOperationState operationState, SetupPhase phase, SetupErrorCode errorCode) throws IOException { + ManagedConfigurationTransaction transaction = mock(ManagedConfigurationTransaction.class); + when(transaction.apply(any())).thenReturn(outcome); + SetupOperationRegistry operations = spy(new SetupOperationRegistry(Clock.systemUTC())); + SetupConfigurationCoordinator coordinator = new SetupConfigurationCoordinator(transaction, operations); + + SetupApiException failure = assertThrows(SetupApiException.class, + () -> configure(transport, coordinator, SetupPhase.CONFIGURATION_REQUIRED, + ApplyMode.MANAGED_WRITE, managedCapability())); + + assertEquals(errorCode, failure.errorCode(), transport.name()); + assertFinishedOperation(operations, operationState, phase, errorCode); + } + + private static void assertFinishedOperation( + SetupOperationRegistry operations, SetupOperationState state, + SetupPhase phase, SetupErrorCode errorCode) { + var operation = org.mockito.ArgumentCaptor.forClass(String.class); + verify(operations).finish(operation.capture(), eq(state), eq(phase), eq(errorCode), eq(false)); + OperationResponse recorded = operations.get(operation.getValue()); + assertEquals(state, recorded.state()); + assertEquals(phase, recorded.phase()); + assertEquals(errorCode, recorded.errorCode()); + } + + private static org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationResponse configure( + Transport transport, SetupConfigurationCoordinator coordinator, SetupPhase phase, + ApplyMode applyMode, ManagedConfigCapability capability) { + if (transport == Transport.BROWSER) { + return coordinator.configure(request(phase, applyMode), capability); + } + try (SecretValue password = SecretValue.of("metadata-password")) { + return coordinator.configure(headlessRequest(phase, applyMode, password), capability); + } + } + + private static HeadlessSetupWorkflow.RequiredConfiguration headlessRequest( + SetupPhase phase, ApplyMode applyMode, SecretValue password) { + return new HeadlessSetupWorkflow.RequiredConfiguration(phase, applyMode, + new HeadlessSetupWorkflow.Metadata(MetadataDatabaseKind.H2, + "jdbc:h2:./data/setup", "sa", password), + new HeadlessSetupWorkflow.Telemetry("localhost:4001", "http://localhost:4000", + "public", Optional.empty(), Optional.empty())); + } + + private static ManagedConfigCapability managedCapability() { + return new ManagedConfigCapability(ApplyMode.MANAGED_WRITE, true, DeploymentConstraint.NONE); + } + + private static ManagedConfigCapability externalCapability() { + return new ManagedConfigCapability( + ApplyMode.EXTERNAL_APPLY, false, DeploymentConstraint.READ_ONLY); + } + + private enum Transport { + BROWSER, + HEADLESS + } + private static ConfigurationRequest request(ApplyMode applyMode) { return request(SetupPhase.CONFIGURATION_REQUIRED, applyMode); } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionServiceTest.java index f142bd631e..e5e0a0c393 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionServiceTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionServiceTest.java @@ -68,7 +68,7 @@ class SetupTransitionServiceTest { SetupConfigurationCoordinator configuration = mock(SetupConfigurationCoordinator.class); when(configuration.configure(any(ConfigurationRequest.class), any())) .thenReturn(configurationResponse("browser")); - when(configuration.configure(any(HeadlessSetupWorkflow.RequiredConfiguration.class), any(), any())) + when(configuration.configure(any(HeadlessSetupWorkflow.RequiredConfiguration.class), any())) .thenReturn(configurationResponse("headless")); SetupTransitionService transitions = transitions(state, validator, configuration, capability, Optional.empty(), Optional.empty()); @@ -171,7 +171,7 @@ class SetupTransitionServiceTest { ConfigurationResponse response = new ConfigurationResponse("replacement", SetupOperationState.AWAITING_EXTERNAL_APPLY, SetupPhase.EXTERNAL_APPLY_REQUIRED, 0, true); when(configuration.configure(any(ConfigurationRequest.class), any())).thenReturn(response); - when(configuration.configure(any(HeadlessSetupWorkflow.RequiredConfiguration.class), any(), any())) + when(configuration.configure(any(HeadlessSetupWorkflow.RequiredConfiguration.class), any())) .thenReturn(response); SetupRuntimeState browserState = state(capability, SetupPhase.EXTERNAL_APPLY_REQUIRED, false, null); @@ -191,7 +191,7 @@ class SetupTransitionServiceTest { assertThat(browserState.phase()).isEqualTo(SetupPhase.EXTERNAL_APPLY_REQUIRED); assertThat(headlessState.phase()).isEqualTo(SetupPhase.EXTERNAL_APPLY_REQUIRED); verify(configuration).configure(any(ConfigurationRequest.class), any()); - verify(configuration).configure(any(HeadlessSetupWorkflow.RequiredConfiguration.class), any(), any()); + verify(configuration).configure(any(HeadlessSetupWorkflow.RequiredConfiguration.class), any()); } @Test From 7d0b84a20c30242eeedb36259926fc2698342cfd Mon Sep 17 00:00:00 2001 From: Logic Date: Sun, 9 Aug 2026 07:18:07 +0800 Subject: [PATCH 25/71] Require setup runtime dependencies --- .../setup/api/SetupApiConfiguration.java | 2 +- .../UnattendedSetupInitializer.java | 16 ++-- .../UnattendedSetupInitializerTest.java | 77 +++++++++++++++++-- .../runtime/StartupModePropertyProbe.java | 4 - .../runtime/StartupModePropertyProbeTest.java | 12 ++- 5 files changed, 86 insertions(+), 25 deletions(-) diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java index b46b145260..3b18ec0907 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java @@ -170,7 +170,7 @@ public class SetupApiConfiguration { public ApplicationRunner unattendedSetupRunner( HeadlessSetupWorkflow workflow, Environment environment, SetupRuntimeTransitionScheduler scheduler) { UnattendedSetupInitializer initializer = new UnattendedSetupInitializer( - workflow, environment, new SetupPasswordFileLoader(), Optional.of(scheduler)); + workflow, environment, new SetupPasswordFileLoader(), scheduler); return arguments -> initializer.initialize(); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializer.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializer.java index d92c614467..2a501dcfb5 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializer.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializer.java @@ -21,6 +21,7 @@ import java.nio.file.Path; import java.util.Arrays; import java.util.List; import java.util.Locale; +import java.util.Objects; import java.util.Optional; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; @@ -42,20 +43,15 @@ public final class UnattendedSetupInitializer { private final HeadlessSetupWorkflow workflow; private final Environment environment; private final SetupPasswordFileLoader passwords; - private final Optional transitions; - - public UnattendedSetupInitializer( - HeadlessSetupWorkflow workflow, Environment environment, SetupPasswordFileLoader passwords) { - this(workflow, environment, passwords, Optional.empty()); - } + private final SetupRuntimeTransitionScheduler transitions; public UnattendedSetupInitializer(HeadlessSetupWorkflow workflow, Environment environment, SetupPasswordFileLoader passwords, - Optional transitions) { + SetupRuntimeTransitionScheduler transitions) { this.workflow = workflow; this.environment = environment; this.passwords = passwords; - this.transitions = transitions; + this.transitions = Objects.requireNonNull(transitions, "transitions"); } public void initialize() { @@ -115,7 +111,7 @@ public final class UnattendedSetupInitializer { new HeadlessSetupWorkflow.RequiredConfiguration( status.phase(), status.applyMode(), metadata, telemetry)); if (response.phase() == SetupPhase.APPLICATION_STARTING) { - transitions.ifPresent(SetupRuntimeTransitionScheduler::configurationApplied); + transitions.configurationApplied(); } } @@ -130,7 +126,7 @@ public final class UnattendedSetupInitializer { private void complete() { workflow.complete(acknowledgedWarnings()); - transitions.ifPresent(SetupRuntimeTransitionScheduler::installationCompleted); + transitions.installationCompleted(); } private List acknowledgedWarnings() { diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializerTest.java index b4201e408b..5e23366acf 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializerTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializerTest.java @@ -22,19 +22,21 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.PosixFilePermissions; import java.time.Instant; -import java.util.Optional; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigSource; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ManagementDatabaseSummary; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionalConfigurationSummary; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupAccess; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.StatusResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; @@ -54,22 +56,27 @@ class UnattendedSetupInitializerTest { void completedRestartIsIdempotentAndPerformsNoWrites() { HeadlessSetupWorkflow workflow = mock(HeadlessSetupWorkflow.class); when(workflow.status()).thenReturn(status(SetupPhase.COMPLETE)); + SetupRuntimeTransitionScheduler transitions = mock(SetupRuntimeTransitionScheduler.class); MockEnvironment environment = new MockEnvironment().withProperty( UnattendedSetupInitializer.ENABLED_PROPERTY, "true"); - new UnattendedSetupInitializer(workflow, environment, new SetupPasswordFileLoader()).initialize(); + new UnattendedSetupInitializer(workflow, environment, new SetupPasswordFileLoader(), transitions) + .initialize(); verify(workflow, never()).configure(any()); verify(workflow, never()).createAdministrator(any(), any()); verify(workflow, never()).complete(any()); + verifyNoInteractions(transitions); } @Test void disabledInitializationDoesNotEvenReadSetupStatus() { HeadlessSetupWorkflow workflow = mock(HeadlessSetupWorkflow.class); + SetupRuntimeTransitionScheduler transitions = mock(SetupRuntimeTransitionScheduler.class); new UnattendedSetupInitializer(workflow, new MockEnvironment(), - new SetupPasswordFileLoader()).initialize(); + new SetupPasswordFileLoader(), transitions).initialize(); verify(workflow, never()).status(); + verifyNoInteractions(transitions); } @Test @@ -86,7 +93,7 @@ class UnattendedSetupInitializerTest { .withProperty("hertzbeat.setup.administrator.username", "operator") .withProperty("hertzbeat.setup.administrator.password-file", passwordFile.toString()); - new UnattendedSetupInitializer(workflow, environment, new SetupPasswordFileLoader(), Optional.of(transitions)) + new UnattendedSetupInitializer(workflow, environment, new SetupPasswordFileLoader(), transitions) .initialize(); verify(workflow).createAdministrator(eq("operator"), any()); @@ -97,21 +104,79 @@ class UnattendedSetupInitializerTest { @Test void explicitlyConfirmsTheSameH2AndPlainHttpWarningsAsBrowserSetup() { HeadlessSetupWorkflow workflow = mock(HeadlessSetupWorkflow.class); + SetupRuntimeTransitionScheduler transitions = mock(SetupRuntimeTransitionScheduler.class); when(workflow.status()).thenReturn(status(SetupPhase.OPTIONAL_CONFIGURATION)); MockEnvironment environment = new MockEnvironment() .withProperty(UnattendedSetupInitializer.ENABLED_PROPERTY, "true") .withProperty("hertzbeat.setup.unattended.acknowledged-warnings", "h2_non_production, server_otlp_plaintext"); - new UnattendedSetupInitializer(workflow, environment, new SetupPasswordFileLoader()).initialize(); + new UnattendedSetupInitializer(workflow, environment, new SetupPasswordFileLoader(), transitions) + .initialize(); verify(workflow).complete(java.util.List.of( SetupWarningCode.H2_NON_PRODUCTION, SetupWarningCode.SERVER_OTLP_PLAINTEXT)); + verify(transitions).installationCompleted(); + } + + @Test + void managedConfigurationSchedulesRestartOnlyAfterApplicationStartingResponse() throws Exception { + HeadlessSetupWorkflow workflow = mock(HeadlessSetupWorkflow.class); + SetupRuntimeTransitionScheduler transitions = mock(SetupRuntimeTransitionScheduler.class); + when(workflow.status()).thenReturn(status(SetupPhase.CONFIGURATION_REQUIRED)); + when(workflow.configure(any())).thenReturn(new ConfigurationResponse( + "operation", SetupOperationState.AWAITING_RESTART, + SetupPhase.APPLICATION_STARTING, 1_000, false)); + MockEnvironment environment = configurationEnvironment("managed-metadata-password"); + + new UnattendedSetupInitializer(workflow, environment, new SetupPasswordFileLoader(), transitions) + .initialize(); + + verify(workflow).configure(any()); + verify(transitions).configurationApplied(); + verify(transitions, never()).installationCompleted(); + } + + @Test + void externalApplyConfigurationDoesNotScheduleRuntimeTransition() throws Exception { + HeadlessSetupWorkflow workflow = mock(HeadlessSetupWorkflow.class); + SetupRuntimeTransitionScheduler transitions = mock(SetupRuntimeTransitionScheduler.class); + when(workflow.status()).thenReturn( + status(SetupPhase.CONFIGURATION_REQUIRED, ApplyMode.EXTERNAL_APPLY)); + when(workflow.configure(any())).thenReturn(new ConfigurationResponse( + "operation", SetupOperationState.AWAITING_EXTERNAL_APPLY, + SetupPhase.EXTERNAL_APPLY_REQUIRED, 1_000, true)); + + new UnattendedSetupInitializer(workflow, + configurationEnvironment("external-metadata-password"), + new SetupPasswordFileLoader(), transitions).initialize(); + + verify(workflow).configure(any()); + verifyNoInteractions(transitions); + } + + private MockEnvironment configurationEnvironment(String fileName) throws Exception { + Path passwordFile = temporaryDirectory.resolve(fileName); + Files.writeString(passwordFile, "metadata-secret\n"); + Files.setPosixFilePermissions(passwordFile, PosixFilePermissions.fromString("rw-------")); + return new MockEnvironment() + .withProperty(UnattendedSetupInitializer.ENABLED_PROPERTY, "true") + .withProperty("hertzbeat.setup.metadata.kind", "h2") + .withProperty("hertzbeat.setup.metadata.jdbc-url", "jdbc:h2:./data/setup") + .withProperty("hertzbeat.setup.metadata.username", "sa") + .withProperty("hertzbeat.setup.metadata.password-file", passwordFile.toString()) + .withProperty("hertzbeat.setup.telemetry.grpc-endpoints", "localhost:4001") + .withProperty("hertzbeat.setup.telemetry.http-endpoint", "http://localhost:4000") + .withProperty("hertzbeat.setup.telemetry.database", "public"); } private static StatusResponse status(SetupPhase phase) { + return status(phase, ApplyMode.MANAGED_WRITE); + } + + private static StatusResponse status(SetupPhase phase, ApplyMode applyMode) { return new StatusResponse(phase, Instant.parse("2026-08-08T00:00:00Z"), SetupAccess.LOCAL, - ApplyMode.MANAGED_WRITE, true, null, null, + applyMode, applyMode == ApplyMode.MANAGED_WRITE, null, null, new ManagementDatabaseSummary(MetadataDatabaseKind.H2, true, ConfigSource.UI_MANAGED, false), new TelemetryStoreSummary(TelemetryStoreKind.GREPTIME, true, ConfigSource.UI_MANAGED, false), phase != SetupPhase.ADMINISTRATOR_REQUIRED, diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbe.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbe.java index c6679bdb26..f974ad29c6 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbe.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbe.java @@ -28,10 +28,6 @@ public final class StartupModePropertyProbe implements StartupDecisionProbe { private final StartupDecisionProbe fallback; - public StartupModePropertyProbe() { - this(ignored -> StartupDecision.normal()); - } - public StartupModePropertyProbe(StartupDecisionProbe fallback) { this.fallback = Objects.requireNonNull(fallback, "fallback"); } diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbeTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbeTest.java index eb4d4cc642..ef384f2a1f 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbeTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbeTest.java @@ -27,14 +27,14 @@ class StartupModePropertyProbeTest { @Test void missingOverridePreservesNormalStartup() { - StartupDecision decision = new StartupModePropertyProbe().decide(new String[0], null, null); + StartupDecision decision = normalProbe().decide(new String[0], null, null); assertEquals(RuntimeMode.NORMAL, decision.mode()); } @Test void systemPropertyTakesPrecedenceOverEnvironment() { - StartupDecision decision = new StartupModePropertyProbe().decide( + StartupDecision decision = normalProbe().decide( new String[0], "full_setup_gated", "setup_only"); assertEquals(RuntimeMode.FULL_SETUP_GATED, decision.mode()); @@ -42,14 +42,14 @@ class StartupModePropertyProbeTest { @Test void environmentSelectsSetupOnlyWhenSystemPropertyIsMissing() { - StartupDecision decision = new StartupModePropertyProbe().decide(new String[0], null, "setup_only"); + StartupDecision decision = normalProbe().decide(new String[0], null, "setup_only"); assertEquals(RuntimeMode.SETUP_ONLY, decision.mode()); } @Test void invalidOverrideFailsClosedForCoordinatorRecovery() { - StartupModePropertyProbe probe = new StartupModePropertyProbe(); + StartupModePropertyProbe probe = normalProbe(); assertThrows(IllegalArgumentException.class, () -> probe.decide(new String[0], "unsupported", null)); @@ -62,4 +62,8 @@ class StartupModePropertyProbeTest { assertEquals(expected, new StartupModePropertyProbe(fallback).decide(new String[0], null, null)); } + + private static StartupModePropertyProbe normalProbe() { + return new StartupModePropertyProbe(ignored -> StartupDecision.normal()); + } } From 2702c55e73d7e033782ad3e093a2ceefe551723e Mon Sep 17 00:00:00 2001 From: Logic Date: Sun, 9 Aug 2026 08:54:31 +0800 Subject: [PATCH 26/71] Add operator-owned setup public addresses --- .../manager/setup/api/SetupApiContract.java | 28 ++- .../api/SetupStatusProjectionFactory.java | 20 ++- .../ApplicationConfigDocumentCodec.java | 34 ++-- .../config/ManagedConfigurationKeys.java | 1 + .../config/ManagedOptionalConfiguration.java | 41 ++--- .../setup/config/SetupPublicAddress.java | 159 ++++++++++++++++++ .../OptionalConfigurationProjection.java | 21 ++- .../PublicAccessConfigurationValidator.java | 46 +++++ ...InstrumentationConfigurationValidator.java | 59 ------- .../SetupConfigurationProjection.java | 2 +- .../workflow/SetupOptionsCoordinator.java | 19 ++- .../setup/workflow/SetupRequestValidator.java | 7 +- .../workflow/SetupTransitionService.java | 6 +- .../setup/workflow/SetupWarningPolicy.java | 31 ++-- .../setup/api/SetupApiContractTest.java | 41 +++-- .../setup/api/SetupControllerTest.java | 2 +- .../api/SetupStatusProjectionFactoryTest.java | 23 ++- ...dOptionalConfigurationPersistenceTest.java | 71 ++++++-- .../UnattendedSetupInitializerTest.java | 6 +- .../workflow/DefaultSetupWorkflowTest.java | 9 +- .../HeadlessSetupCoordinatorTest.java | 6 +- .../workflow/SetupRequestValidatorTest.java | 139 ++++++++++++--- .../workflow/SetupTransitionServiceTest.java | 18 +- .../workflow/SetupWarningPolicyTest.java | 11 +- 24 files changed, 579 insertions(+), 221 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SetupPublicAddress.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/PublicAccessConfigurationValidator.java delete mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ServerInstrumentationConfigurationValidator.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java index aa51c5a3fe..717b1564a5 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java @@ -193,7 +193,7 @@ public final class SetupApiContract { public enum ValidationSection implements WireValue { METADATA_DATABASE("metadata_database"), TELEMETRY_STORE("telemetry_store"), - SERVER_INSTRUMENTATION("server_instrumentation"), + PUBLIC_ACCESS("public_access"), MAIL("mail"); private final String value; @@ -244,7 +244,7 @@ public final class SetupApiContract { METADATA_SCHEMA_MISMATCH("metadata_schema_mismatch"), METADATA_INSUFFICIENT_PRIVILEGES("metadata_insufficient_privileges"), TELEMETRY_CONNECTION_FAILED("telemetry_connection_failed"), - SERVER_INSTRUMENTATION_INVALID("server_instrumentation_invalid"), + PUBLIC_ADDRESS_INVALID("public_address_invalid"), MAIL_CONNECTION_FAILED("mail_connection_failed"), ADMINISTRATOR_ALREADY_CONFIGURED("administrator_already_configured"), ADMINISTRATOR_USERNAME_INVALID("administrator_username_invalid"), @@ -274,7 +274,7 @@ public final class SetupApiContract { public enum SetupWarningCode implements WireValue { EXTERNAL_APPLY_REQUIRED("external_apply_required"), RESTART_REQUIRED("restart_required"), - SERVER_OTLP_PLAINTEXT("server_otlp_plaintext"), + PUBLIC_ADDRESS_PLAINTEXT("public_address_plaintext"), MAIL_SECURITY_NONE("mail_security_none"), H2_NON_PRODUCTION("h2_non_production"); @@ -338,6 +338,7 @@ public final class SetupApiContract { /** Secret-free optional configuration status. */ public record OptionalConfigurationSummary( + boolean publicBaseUrlConfigured, boolean serverOtlpHttpConfigured, boolean serverOtlpGrpcConfigured, boolean retentionConfigured, @@ -407,10 +408,18 @@ public final class SetupApiContract { } } - /** Server OTLP endpoint input; HTTP and HTTPS are both contractually valid. */ - public record ServerInstrumentationConfiguration( + /** Operator-owned public addresses; values are never inferred from the setup request. */ + public record PublicAccessConfiguration( + String publicBaseUrl, String serverOtlpHttpEndpoint, String serverOtlpGrpcEndpoint) { + + @Override + public String toString() { + return "PublicAccessConfiguration[publicBaseUrlProvided=" + hasText(publicBaseUrl) + + ", serverOtlpHttpEndpointProvided=" + hasText(serverOtlpHttpEndpoint) + + ", serverOtlpGrpcEndpointProvided=" + hasText(serverOtlpGrpcEndpoint) + "]"; + } } /** Mail input. */ @@ -440,16 +449,16 @@ public final class SetupApiContract { @NotNull ValidationSection section, @Valid MetadataDatabaseConfiguration managementDatabase, @Valid TelemetryStoreConfiguration telemetryStore, - @Valid ServerInstrumentationConfiguration serverInstrumentation, + @Valid PublicAccessConfiguration publicAccess, @Valid MailConfiguration mail) { public ValidateRequest { Objects.requireNonNull(section, "section"); - int supplied = countPresent(managementDatabase, telemetryStore, serverInstrumentation, mail); + int supplied = countPresent(managementDatabase, telemetryStore, publicAccess, mail); boolean matches = switch (section) { case METADATA_DATABASE -> managementDatabase != null; case TELEMETRY_STORE -> telemetryStore != null; - case SERVER_INSTRUMENTATION -> serverInstrumentation != null; + case PUBLIC_ACCESS -> publicAccess != null; case MAIL -> mail != null; }; if (supplied != 1 || !matches) { @@ -527,13 +536,14 @@ public final class SetupApiContract { /** Optional setup input. */ public record OptionsRequest( - @Valid ServerInstrumentationConfiguration serverInstrumentation, + @Valid PublicAccessConfiguration publicAccess, @Valid RetentionConfiguration retention, @Valid MailConfiguration mail) { } /** Secret-free optional setup result. */ public record OptionsResponse( + boolean publicBaseUrlConfigured, boolean serverOtlpHttpConfigured, boolean serverOtlpGrpcConfigured, boolean retentionConfigured, diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupStatusProjectionFactory.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupStatusProjectionFactory.java index 26fbbcd2bf..8bb8318634 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupStatusProjectionFactory.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupStatusProjectionFactory.java @@ -13,6 +13,7 @@ import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.MAIL_HOST; import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.MAIL_SSL_ENABLED; import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.MAIL_STARTTLS_ENABLED; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.PUBLIC_BASE_URL; import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.SERVER_OTLP_GRPC; import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.SERVER_OTLP_HTTP; @@ -27,8 +28,8 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreSum import org.apache.hertzbeat.manager.setup.config.EffectiveConfigurationResolver; import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector.Inspection; import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector.State; -import org.apache.hertzbeat.manager.setup.config.ManagedOptionalConfiguration.ServerInstrumentationSettings; import org.apache.hertzbeat.manager.setup.config.RestartRequirement; +import org.apache.hertzbeat.manager.setup.config.SetupPublicAddress; import org.apache.hertzbeat.manager.setup.workflow.SetupConfigurationProjection; import org.apache.hertzbeat.manager.setup.workflow.SetupWarningPolicy; import org.springframework.core.env.Environment; @@ -49,8 +50,11 @@ final class SetupStatusProjectionFactory { MetadataDatabaseKind kind = MetadataDatabaseKind.valueOf(database.value().toUpperCase(Locale.ROOT)); boolean mailConfigured = externallyConfigured(environment, MAIL_HOST, true); OptionalConfigurationSummary optional = new OptionalConfigurationSummary( - externallyConfiguredEndpoint(environment, SERVER_OTLP_HTTP), - externallyConfiguredEndpoint(environment, SERVER_OTLP_GRPC), + externallyConfiguredAddress(environment, PUBLIC_BASE_URL, SetupPublicAddress.Kind.PUBLIC_BASE_URL), + externallyConfiguredAddress( + environment, SERVER_OTLP_HTTP, SetupPublicAddress.Kind.SERVER_OTLP_ENDPOINT), + externallyConfiguredAddress( + environment, SERVER_OTLP_GRPC, SetupPublicAddress.Kind.SERVER_OTLP_ENDPOINT), externallyConfigured(environment, GREPTIME_EXPIRE_TIME, true), mailConfigured); MailSecurity mailSecurity = mailConfigured ? mailSecurity(environment) : null; return new SetupConfigurationProjection( @@ -59,7 +63,7 @@ final class SetupStatusProjectionFactory { new TelemetryStoreSummary(TelemetryStoreKind.GREPTIME, managedPresent || telemetrySource != ConfigSource.BUILT_IN_DEFAULT, telemetrySource, false), optional, SetupWarningPolicy.INSTANCE.evaluate( - kind, environment.getProperty(SERVER_OTLP_HTTP), + kind, environment.getProperty(PUBLIC_BASE_URL), environment.getProperty(SERVER_OTLP_HTTP), environment.getProperty(SERVER_OTLP_GRPC), mailSecurity)); } @@ -72,13 +76,15 @@ final class SetupStatusProjectionFactory { && (!requireText || !resolved.value().isBlank()); } - private boolean externallyConfiguredEndpoint(Environment environment, String key) { + private boolean externallyConfiguredAddress(Environment environment, String key, SetupPublicAddress.Kind kind) { if (!environment.containsProperty(key)) { return false; } var resolved = resolver.resolve(environment, key, RestartRequirement.LIVE_RELOAD); - return resolved.source() != ConfigSource.BUILT_IN_DEFAULT - && ServerInstrumentationSettings.normalize(resolved.value()).isPresent(); + boolean valid = kind == SetupPublicAddress.Kind.PUBLIC_BASE_URL + ? SetupPublicAddress.tryPublicBaseUrl(resolved.value()).isPresent() + : SetupPublicAddress.tryServerOtlpEndpoint(resolved.value()).isPresent(); + return resolved.source() != ConfigSource.BUILT_IN_DEFAULT && valid; } private static MailSecurity mailSecurity(Environment environment) { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ApplicationConfigDocumentCodec.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ApplicationConfigDocumentCodec.java index e489a362d0..f7e2284fd7 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ApplicationConfigDocumentCodec.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ApplicationConfigDocumentCodec.java @@ -30,6 +30,7 @@ import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.MAIL_FROM_ADDRESS; import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.MAIL_SSL_ENABLED; import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.MAIL_STARTTLS_ENABLED; +import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.PUBLIC_BASE_URL; import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.SERVER_OTLP_GRPC; import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.SERVER_OTLP_HTTP; import static org.apache.hertzbeat.manager.setup.config.ManagedConfigurationKeys.SERVER_AUTHENTICATION; @@ -62,7 +63,7 @@ final class ApplicationConfigDocumentCodec implements ManagedDocumentCodec values.put(GREPTIME_USERNAME, username)); - value.optional().serverInstrumentation().ifPresent(instrumentation -> { - values.put(SERVER_PROFILE_ID, MANAGED_SERVER_PROFILE_ID); - values.put(SERVER_AUTHENTICATION, MANAGED_SERVER_AUTHENTICATION); - instrumentation.serverOtlpHttpEndpoint().ifPresent(item -> values.put(SERVER_OTLP_HTTP, item)); - instrumentation.serverOtlpGrpcEndpoint().ifPresent(item -> values.put(SERVER_OTLP_GRPC, item)); + value.optional().publicAccess().ifPresent(publicAccess -> { + publicAccess.publicBaseUrl().ifPresent(item -> values.put(PUBLIC_BASE_URL, item)); + if (publicAccess.serverOtlpHttpEndpoint().isPresent() + || publicAccess.serverOtlpGrpcEndpoint().isPresent()) { + values.put(SERVER_PROFILE_ID, MANAGED_SERVER_PROFILE_ID); + values.put(SERVER_AUTHENTICATION, MANAGED_SERVER_AUTHENTICATION); + } + publicAccess.serverOtlpHttpEndpoint().ifPresent(item -> values.put(SERVER_OTLP_HTTP, item)); + publicAccess.serverOtlpGrpcEndpoint().ifPresent(item -> values.put(SERVER_OTLP_GRPC, item)); }); value.optional().retention().ifPresent(retention -> values.put(GREPTIME_EXPIRE_TIME, retention.days() + "d")); @@ -129,7 +134,7 @@ final class ApplicationConfigDocumentCodec implements ManagedDocumentCodec values) { - boolean instrumentationPresent = containsAny(values, SERVER_OTLP_HTTP, SERVER_OTLP_GRPC); - Optional instrumentation = - instrumentationPresent ? Optional.of(new ManagedOptionalConfiguration.ServerInstrumentationSettings( - optionalText(values, SERVER_OTLP_HTTP), optionalText(values, SERVER_OTLP_GRPC))) + boolean publicAccessPresent = containsAny(values, PUBLIC_BASE_URL, SERVER_OTLP_HTTP, SERVER_OTLP_GRPC); + Optional publicAccess = + publicAccessPresent ? Optional.of(new ManagedOptionalConfiguration.PublicAccessSettings( + optionalText(values, PUBLIC_BASE_URL), optionalText(values, SERVER_OTLP_HTTP), + optionalText(values, SERVER_OTLP_GRPC))) : Optional.empty(); Optional retention = values.containsKey(GREPTIME_EXPIRE_TIME) ? Optional.of(new ManagedOptionalConfiguration.RetentionSettings(retentionDays(values))) @@ -166,7 +172,7 @@ final class ApplicationConfigDocumentCodec implements ManagedDocumentCodec values) { @@ -176,7 +182,7 @@ final class ApplicationConfigDocumentCodec implements ManagedDocumentCodec values) { + private static boolean completeServerOtlpGroup(Map values) { boolean endpointKeyPresent = containsAny(values, SERVER_OTLP_HTTP, SERVER_OTLP_GRPC); boolean endpointPresent = meaningfulEndpoint(values, SERVER_OTLP_HTTP) || meaningfulEndpoint(values, SERVER_OTLP_GRPC); @@ -193,7 +199,7 @@ final class ApplicationConfigDocumentCodec implements ManagedDocumentCodec values, String key) { return values.get(key) instanceof String endpoint - && ManagedOptionalConfiguration.ServerInstrumentationSettings.normalize(endpoint).isPresent(); + && SetupPublicAddress.tryServerOtlpEndpoint(endpoint).isPresent(); } private static boolean containsAny(Map values, String... keys) { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationKeys.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationKeys.java index 7ec81ac4a9..88263c0772 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationKeys.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationKeys.java @@ -29,6 +29,7 @@ public final class ManagedConfigurationKeys { public static final String GREPTIME_DATABASE = "warehouse.store.greptime.database"; public static final String GREPTIME_USERNAME = "warehouse.store.greptime.username"; public static final String GREPTIME_PASSWORD = "warehouse.store.greptime.password"; + public static final String PUBLIC_BASE_URL = "hertzbeat.setup.public-base-url"; public static final String SERVER_OTLP_HTTP = "hertzbeat.instrumentation.server.otlp-http-endpoint"; public static final String SERVER_OTLP_GRPC = "hertzbeat.instrumentation.server.otlp-grpc-endpoint"; public static final String SERVER_PROFILE_ID = "hertzbeat.instrumentation.server.profile-id"; diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionalConfiguration.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionalConfiguration.java index 00631ec179..39ab3d9357 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionalConfiguration.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionalConfiguration.java @@ -23,12 +23,12 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; /** Typed optional setup overlay kept in the existing managed application document. */ public record ManagedOptionalConfiguration( - Optional serverInstrumentation, + Optional publicAccess, Optional retention, Optional mail) { public ManagedOptionalConfiguration { - Objects.requireNonNull(serverInstrumentation, "serverInstrumentation"); + Objects.requireNonNull(publicAccess, "publicAccess"); Objects.requireNonNull(retention, "retention"); Objects.requireNonNull(mail, "mail"); } @@ -37,33 +37,36 @@ public record ManagedOptionalConfiguration( return new ManagedOptionalConfiguration(Optional.empty(), Optional.empty(), Optional.empty()); } - /** Optional server OTLP intake endpoints. */ - public record ServerInstrumentationSettings( + /** Explicit operator-owned public access addresses. */ + public record PublicAccessSettings( + Optional publicBaseUrl, Optional serverOtlpHttpEndpoint, Optional serverOtlpGrpcEndpoint) { - public ServerInstrumentationSettings { + public PublicAccessSettings { + Objects.requireNonNull(publicBaseUrl, "publicBaseUrl"); Objects.requireNonNull(serverOtlpHttpEndpoint, "serverOtlpHttpEndpoint"); Objects.requireNonNull(serverOtlpGrpcEndpoint, "serverOtlpGrpcEndpoint"); - serverOtlpHttpEndpoint = normalizeConfigured(serverOtlpHttpEndpoint); - serverOtlpGrpcEndpoint = normalizeConfigured(serverOtlpGrpcEndpoint); - if (serverOtlpHttpEndpoint.isEmpty() && serverOtlpGrpcEndpoint.isEmpty()) { - throw new IllegalArgumentException("At least one server instrumentation endpoint is required"); + publicBaseUrl = validateConfigured(publicBaseUrl, SetupPublicAddress.Kind.PUBLIC_BASE_URL); + serverOtlpHttpEndpoint = validateConfigured( + serverOtlpHttpEndpoint, SetupPublicAddress.Kind.SERVER_OTLP_ENDPOINT); + serverOtlpGrpcEndpoint = validateConfigured( + serverOtlpGrpcEndpoint, SetupPublicAddress.Kind.SERVER_OTLP_ENDPOINT); + if (publicBaseUrl.isEmpty() && serverOtlpHttpEndpoint.isEmpty() && serverOtlpGrpcEndpoint.isEmpty()) { + throw new IllegalArgumentException("At least one public access address is required"); } } - public static Optional normalize(String value) { - if (value == null) { + private static Optional validateConfigured(Optional endpoint, SetupPublicAddress.Kind kind) { + if (endpoint.isEmpty()) { return Optional.empty(); } - String normalized = value.trim(); - return normalized.isEmpty() ? Optional.empty() : Optional.of(normalized); - } - - private static Optional normalizeConfigured(Optional endpoint) { - if (endpoint.isPresent() && normalize(endpoint.orElseThrow()).isEmpty()) { - throw new IllegalArgumentException("Server instrumentation endpoint must not be blank"); + String value = endpoint.orElseThrow(); + Optional address = kind == SetupPublicAddress.Kind.PUBLIC_BASE_URL + ? SetupPublicAddress.publicBaseUrl(value) : SetupPublicAddress.serverOtlpEndpoint(value); + if (address.isEmpty()) { + throw new IllegalArgumentException("Public access address must not be blank"); } - return endpoint.flatMap(ServerInstrumentationSettings::normalize); + return address.map(SetupPublicAddress::value); } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SetupPublicAddress.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SetupPublicAddress.java new file mode 100644 index 0000000000..7fba49c19c --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SetupPublicAddress.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.net.Inet4Address; +import java.net.InetAddress; +import java.net.URI; +import java.util.Locale; +import java.util.Optional; +import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.IntakeEndpoint; + +/** A validated operator-advertised address; validation is purely syntactic and never resolves DNS. */ +public record SetupPublicAddress(String value, Kind kind) { + + /** Address contracts differ between the browser-facing base URL and OTLP intake endpoints. */ + public enum Kind { + PUBLIC_BASE_URL, + SERVER_OTLP_ENDPOINT + } + + public SetupPublicAddress { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException("Public address must not be blank"); + } + value = value.trim(); + URI uri = URI.create(value); + if (kind == null || uri.getHost() == null || uri.getHost().indexOf('%') >= 0 + || wildcardHost(uri.getHost()) || invalidPort(uri.getPort())) { + throw new IllegalArgumentException("Public address is invalid"); + } + if (kind == Kind.PUBLIC_BASE_URL) { + if (!("http".equalsIgnoreCase(uri.getScheme()) || "https".equalsIgnoreCase(uri.getScheme())) + || uri.getUserInfo() != null || uri.getRawQuery() != null || uri.getRawFragment() != null) { + throw new IllegalArgumentException("Public base URL is invalid"); + } + } else { + IntakeEndpoint.fromUrl(value); + } + } + + public static Optional publicBaseUrl(String value) { + return parse(value, Kind.PUBLIC_BASE_URL); + } + + public static Optional serverOtlpEndpoint(String value) { + return parse(value, Kind.SERVER_OTLP_ENDPOINT); + } + + public static Optional tryPublicBaseUrl(String value) { + return tryParse(value, Kind.PUBLIC_BASE_URL); + } + + public static Optional tryServerOtlpEndpoint(String value) { + return tryParse(value, Kind.SERVER_OTLP_ENDPOINT); + } + + public boolean plaintextPublic() { + URI uri = URI.create(value); + return "http".equalsIgnoreCase(uri.getScheme()) && !internalHost(uri.getHost()); + } + + private static Optional parse(String value, Kind kind) { + if (value == null || value.isBlank()) { + return Optional.empty(); + } + return Optional.of(new SetupPublicAddress(value, kind)); + } + + private static Optional tryParse(String value, Kind kind) { + try { + return parse(value, kind); + } catch (IllegalArgumentException failure) { + return Optional.empty(); + } + } + + private static boolean invalidPort(int port) { + return port == 0 || port > 65_535; + } + + private static boolean wildcardHost(String value) { + String host = withoutIpv6Brackets(value.toLowerCase(Locale.ROOT)); + InetAddress address = literalAddress(host); + return address != null && address.isAnyLocalAddress(); + } + + private static boolean internalHost(String value) { + String host = withoutIpv6Brackets(value.toLowerCase(Locale.ROOT)); + InetAddress address = literalAddress(host); + if (address instanceof Inet4Address) { + return privateIpv4(address.getHostAddress()); + } + if (address != null) { + return address.isLoopbackAddress() || address.isLinkLocalAddress() + || privateIpv6(address.getHostAddress()); + } + return host.equals("localhost") || host.endsWith(".localhost") || host.endsWith(".local") + || host.endsWith(".internal") || (!host.contains(".") && !host.contains(":")) + || privateIpv4(host); + } + + private static String withoutIpv6Brackets(String host) { + return host.length() > 1 && host.charAt(0) == '[' && host.charAt(host.length() - 1) == ']' + ? host.substring(1, host.length() - 1) : host; + } + + private static boolean privateIpv4(String host) { + String[] parts = host.split("\\.", -1); + if (parts.length != 4) { + return false; + } + try { + int first = octet(parts[0]); + int second = octet(parts[1]); + octet(parts[2]); + octet(parts[3]); + return first == 10 || first == 127 || first == 0 || (first == 169 && second == 254) + || (first == 172 && second >= 16 && second <= 31) || (first == 192 && second == 168); + } catch (IllegalArgumentException failure) { + return false; + } + } + + private static InetAddress literalAddress(String host) { + try { + return InetAddress.ofLiteral(host); + } catch (IllegalArgumentException failure) { + return null; + } + } + + private static int octet(String value) { + int parsed = Integer.parseInt(value); + if (parsed < 0 || parsed > 255) { + throw new IllegalArgumentException("Invalid IPv4 octet"); + } + return parsed; + } + + private static boolean privateIpv6(String host) { + return host.equals("::1") || host.equals("0:0:0:0:0:0:0:1") || host.startsWith("fc") + || host.startsWith("fd") || host.matches("fe[89ab].*"); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/OptionalConfigurationProjection.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/OptionalConfigurationProjection.java index 00e59d8497..ff4ec83383 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/OptionalConfigurationProjection.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/OptionalConfigurationProjection.java @@ -14,7 +14,7 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsRequest; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; -import org.apache.hertzbeat.manager.setup.config.ManagedOptionalConfiguration.ServerInstrumentationSettings; +import org.apache.hertzbeat.manager.setup.config.SetupPublicAddress; /** Projects persisted optional settings into the secret-free runtime and response shape. */ record OptionalConfigurationProjection( @@ -22,19 +22,22 @@ record OptionalConfigurationProjection( static OptionalConfigurationProjection from(MetadataDatabaseKind databaseKind, OptionsRequest request) { OptionalConfigurationSummary summary = new OptionalConfigurationSummary( - request.serverInstrumentation() != null - && ServerInstrumentationSettings.normalize( - request.serverInstrumentation().serverOtlpHttpEndpoint()).isPresent(), - request.serverInstrumentation() != null - && ServerInstrumentationSettings.normalize( - request.serverInstrumentation().serverOtlpGrpcEndpoint()).isPresent(), + request.publicAccess() != null + && SetupPublicAddress.tryPublicBaseUrl(request.publicAccess().publicBaseUrl()).isPresent(), + request.publicAccess() != null + && SetupPublicAddress.tryServerOtlpEndpoint( + request.publicAccess().serverOtlpHttpEndpoint()).isPresent(), + request.publicAccess() != null + && SetupPublicAddress.tryServerOtlpEndpoint( + request.publicAccess().serverOtlpGrpcEndpoint()).isPresent(), request.retention() != null, request.mail() != null); return new OptionalConfigurationProjection( summary, SetupWarningPolicy.INSTANCE.evaluate(databaseKind, request)); } OptionsResponse response() { - return new OptionsResponse(summary.serverOtlpHttpConfigured(), summary.serverOtlpGrpcConfigured(), - summary.retentionConfigured(), summary.mailConfigured(), SetupPhase.OPTIONAL_CONFIGURATION); + return new OptionsResponse(summary.publicBaseUrlConfigured(), summary.serverOtlpHttpConfigured(), + summary.serverOtlpGrpcConfigured(), summary.retentionConfigured(), summary.mailConfigured(), + SetupPhase.OPTIONAL_CONFIGURATION); } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/PublicAccessConfigurationValidator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/PublicAccessConfigurationValidator.java new file mode 100644 index 0000000000..7c599fa69b --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/PublicAccessConfigurationValidator.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.List; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.PublicAccessConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; +import org.apache.hertzbeat.manager.setup.config.SetupPublicAddress; +import org.apache.hertzbeat.manager.setup.workflow.MetadataConfigurationValidator.Validation; + +/** Validates explicit public addresses without consulting request origin, host headers, or network state. */ +final class PublicAccessConfigurationValidator { + Validation validate(PublicAccessConfiguration configuration) { + try { + var publicBaseUrl = SetupPublicAddress.publicBaseUrl(configuration.publicBaseUrl()); + var http = SetupPublicAddress.serverOtlpEndpoint(configuration.serverOtlpHttpEndpoint()); + var grpc = SetupPublicAddress.serverOtlpEndpoint(configuration.serverOtlpGrpcEndpoint()); + if (publicBaseUrl.isEmpty() && http.isEmpty() && grpc.isEmpty()) { + return Validation.failed(SetupErrorCode.PUBLIC_ADDRESS_INVALID); + } + List warnings = publicBaseUrl.filter(SetupPublicAddress::plaintextPublic).isPresent() + || http.filter(SetupPublicAddress::plaintextPublic).isPresent() + || grpc.filter(SetupPublicAddress::plaintextPublic).isPresent() + ? List.of(SetupWarningCode.PUBLIC_ADDRESS_PLAINTEXT) : List.of(); + return new Validation(true, null, warnings); + } catch (IllegalArgumentException failure) { + return Validation.failed(SetupErrorCode.PUBLIC_ADDRESS_INVALID); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ServerInstrumentationConfigurationValidator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ServerInstrumentationConfigurationValidator.java deleted file mode 100644 index f42eb90ff3..0000000000 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ServerInstrumentationConfigurationValidator.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hertzbeat.manager.setup.workflow; - -import java.util.List; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ServerInstrumentationConfiguration; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; -import org.apache.hertzbeat.manager.setup.config.ManagedOptionalConfiguration.ServerInstrumentationSettings; -import org.apache.hertzbeat.manager.setup.workflow.MetadataConfigurationValidator.Validation; -import org.apache.hertzbeat.observability.instrumentation.v2.api.InstrumentationIntakeProfileV2.IntakeEndpoint; - -/** Validates optional server OTLP intake endpoints. */ -final class ServerInstrumentationConfigurationValidator { - Validation validate(ServerInstrumentationConfiguration configuration) { - String http = ServerInstrumentationSettings.normalize( - configuration.serverOtlpHttpEndpoint()).orElse(null); - String grpc = ServerInstrumentationSettings.normalize( - configuration.serverOtlpGrpcEndpoint()).orElse(null); - if ((http == null && grpc == null) || !validEndpoint(http) || !validEndpoint(grpc)) { - return Validation.failed(SetupErrorCode.SERVER_INSTRUMENTATION_INVALID); - } - List warnings = plaintext(http) || plaintext(grpc) - ? List.of(SetupWarningCode.SERVER_OTLP_PLAINTEXT) : List.of(); - return new Validation(true, null, warnings); - } - - private static boolean validEndpoint(String value) { - if (value == null || value.isBlank()) { - return true; - } - try { - IntakeEndpoint.fromUrl(value); - return true; - } catch (IllegalArgumentException failure) { - return false; - } - } - - private static boolean plaintext(String value) { - return value != null && value.regionMatches(true, 0, "http://", 0, 7); - } - -} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationProjection.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationProjection.java index 6e9d7d7858..bda9badc55 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationProjection.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupConfigurationProjection.java @@ -33,7 +33,7 @@ public record SetupConfigurationProjection( ConfigSource.BUILT_IN_DEFAULT, false), new TelemetryStoreSummary(TelemetryStoreKind.GREPTIME, false, ConfigSource.BUILT_IN_DEFAULT, false), - new OptionalConfigurationSummary(false, false, false, false), + new OptionalConfigurationSummary(false, false, false, false, false), List.of(SetupWarningCode.H2_NON_PRODUCTION)); } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOptionsCoordinator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOptionsCoordinator.java index 1dce88e28a..09fb208207 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOptionsCoordinator.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupOptionsCoordinator.java @@ -25,6 +25,7 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiException; import org.apache.hertzbeat.manager.setup.config.ManagedConfigurationTransaction; import org.apache.hertzbeat.manager.setup.config.ManagedOptionalConfiguration; import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.apache.hertzbeat.manager.setup.config.SetupPublicAddress; import org.springframework.http.HttpStatus; /** Maps and atomically persists optional setup settings through the existing two-file transaction. */ @@ -37,14 +38,16 @@ public final class SetupOptionsCoordinator { public void persist(OptionsRequest request) { ManagedOptionalConfiguration options = new ManagedOptionalConfiguration( - Optional.ofNullable(request.serverInstrumentation()).flatMap(value -> { - Optional httpEndpoint = ManagedOptionalConfiguration.ServerInstrumentationSettings - .normalize(value.serverOtlpHttpEndpoint()); - Optional grpcEndpoint = ManagedOptionalConfiguration.ServerInstrumentationSettings - .normalize(value.serverOtlpGrpcEndpoint()); - return httpEndpoint.isEmpty() && grpcEndpoint.isEmpty() ? Optional.empty() - : Optional.of(new ManagedOptionalConfiguration.ServerInstrumentationSettings( - httpEndpoint, grpcEndpoint)); + Optional.ofNullable(request.publicAccess()).flatMap(value -> { + Optional publicBaseUrl = SetupPublicAddress.publicBaseUrl(value.publicBaseUrl()) + .map(SetupPublicAddress::value); + Optional httpEndpoint = SetupPublicAddress + .serverOtlpEndpoint(value.serverOtlpHttpEndpoint()).map(SetupPublicAddress::value); + Optional grpcEndpoint = SetupPublicAddress + .serverOtlpEndpoint(value.serverOtlpGrpcEndpoint()).map(SetupPublicAddress::value); + return publicBaseUrl.isEmpty() && httpEndpoint.isEmpty() && grpcEndpoint.isEmpty() + ? Optional.empty() : Optional.of(new ManagedOptionalConfiguration.PublicAccessSettings( + publicBaseUrl, httpEndpoint, grpcEndpoint)); }), Optional.ofNullable(request.retention()).map(value -> new ManagedOptionalConfiguration.RetentionSettings(value.days())), diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidator.java index dead900e91..c129bda248 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidator.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidator.java @@ -32,8 +32,7 @@ public final class SetupRequestValidator { private final Clock clock; private final MetadataConfigurationValidator metadata = new MetadataConfigurationValidator(); private final TelemetryConfigurationValidator telemetry = new TelemetryConfigurationValidator(); - private final ServerInstrumentationConfigurationValidator serverInstrumentation = - new ServerInstrumentationConfigurationValidator(); + private final PublicAccessConfigurationValidator publicAccess = new PublicAccessConfigurationValidator(); private final MailConfigurationValidator mail = new MailConfigurationValidator(); private final MetadataConnectionProbe metadataConnection; private final TelemetryConnectionProbe telemetryConnection; @@ -58,7 +57,7 @@ public final class SetupRequestValidator { Validation structural = switch (request.section()) { case METADATA_DATABASE -> metadata.validate(request.managementDatabase()); case TELEMETRY_STORE -> telemetry.validate(request.telemetryStore()); - case SERVER_INSTRUMENTATION -> serverInstrumentation.validate(request.serverInstrumentation()); + case PUBLIC_ACCESS -> publicAccess.validate(request.publicAccess()); case MAIL -> mail.validate(request.mail()); }; Validation result = structural.valid() ? liveValidation(request, structural) : structural; @@ -87,7 +86,7 @@ public final class SetupRequestValidator { case METADATA_DATABASE -> metadataConnection.probe(request.managementDatabase()); case TELEMETRY_STORE -> telemetryConnection.probe(request.telemetryStore()); case MAIL -> mailConnection.probe(request.mail()); - case SERVER_INSTRUMENTATION -> Optional.empty(); + case PUBLIC_ACCESS -> Optional.empty(); }; return failure.map(Validation::failed).orElse(structural); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionService.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionService.java index 6990c2fb83..337281358c 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionService.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionService.java @@ -91,9 +91,9 @@ public final class SetupTransitionService { public OptionsResponse configureOptions(OptionsRequest request) { requireWritable(); state.ensurePhase(SetupPhase.OPTIONAL_CONFIGURATION); - if (request.serverInstrumentation() != null) { - requireValid(validator, new ValidateRequest(ValidationSection.SERVER_INSTRUMENTATION, - null, null, request.serverInstrumentation(), null)); + if (request.publicAccess() != null) { + requireValid(validator, new ValidateRequest(ValidationSection.PUBLIC_ACCESS, + null, null, request.publicAccess(), null)); } if (request.mail() != null) { requireValid(validator, new ValidateRequest(ValidationSection.MAIL, diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupWarningPolicy.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupWarningPolicy.java index 6ba9780837..85184f1cfc 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupWarningPolicy.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupWarningPolicy.java @@ -23,7 +23,7 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsRequest; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; -import org.apache.hertzbeat.manager.setup.config.ManagedOptionalConfiguration.ServerInstrumentationSettings; +import org.apache.hertzbeat.manager.setup.config.SetupPublicAddress; /** Single warning policy shared by live setup and restart status projection. */ public final class SetupWarningPolicy { @@ -33,33 +33,32 @@ public final class SetupWarningPolicy { } public List evaluate(MetadataDatabaseKind kind, OptionsRequest options) { - String otlpHttpEndpoint = options.serverInstrumentation() == null ? null - : options.serverInstrumentation().serverOtlpHttpEndpoint(); - String otlpGrpcEndpoint = options.serverInstrumentation() == null ? null - : options.serverInstrumentation().serverOtlpGrpcEndpoint(); + String publicBaseUrl = options.publicAccess() == null ? null : options.publicAccess().publicBaseUrl(); + String otlpHttpEndpoint = options.publicAccess() == null ? null + : options.publicAccess().serverOtlpHttpEndpoint(); + String otlpGrpcEndpoint = options.publicAccess() == null ? null + : options.publicAccess().serverOtlpGrpcEndpoint(); MailSecurity mailSecurity = options.mail() == null ? null : options.mail().security(); - return evaluate(kind, otlpHttpEndpoint, otlpGrpcEndpoint, mailSecurity); + return evaluate(kind, publicBaseUrl, otlpHttpEndpoint, otlpGrpcEndpoint, mailSecurity); } public List evaluate( - MetadataDatabaseKind kind, String otlpHttpEndpoint, String otlpGrpcEndpoint, - MailSecurity mailSecurity) { + MetadataDatabaseKind kind, String publicBaseUrl, String otlpHttpEndpoint, + String otlpGrpcEndpoint, MailSecurity mailSecurity) { List warnings = new ArrayList<>(); if (kind == MetadataDatabaseKind.H2) { warnings.add(SetupWarningCode.H2_NON_PRODUCTION); } - if (plaintext(otlpHttpEndpoint) || plaintext(otlpGrpcEndpoint)) { - warnings.add(SetupWarningCode.SERVER_OTLP_PLAINTEXT); + if (SetupPublicAddress.tryPublicBaseUrl(publicBaseUrl).filter(SetupPublicAddress::plaintextPublic).isPresent() + || SetupPublicAddress.tryServerOtlpEndpoint(otlpHttpEndpoint) + .filter(SetupPublicAddress::plaintextPublic).isPresent() + || SetupPublicAddress.tryServerOtlpEndpoint(otlpGrpcEndpoint) + .filter(SetupPublicAddress::plaintextPublic).isPresent()) { + warnings.add(SetupWarningCode.PUBLIC_ADDRESS_PLAINTEXT); } if (mailSecurity == MailSecurity.NONE) { warnings.add(SetupWarningCode.MAIL_SECURITY_NONE); } return List.copyOf(warnings); } - - private static boolean plaintext(String endpoint) { - return ServerInstrumentationSettings.normalize(endpoint) - .filter(value -> value.regionMatches(true, 0, "http://", 0, 7)) - .isPresent(); - } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java index e2e10c698d..421003bd2a 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java @@ -35,7 +35,7 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailConfiguration import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ServerInstrumentationConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.PublicAccessConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupAccess; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; @@ -77,11 +77,11 @@ class SetupApiContractTest { assertWireValues(MetadataDatabaseKind.values(), "h2", "mysql", "postgresql"); assertWireValues(TelemetryStoreKind.values(), "greptime"); assertWireValues(ValidationSection.values(), "metadata_database", "telemetry_store", - "server_instrumentation", "mail"); + "public_access", "mail"); assertWireValues(MailSecurity.values(), "none", "starttls", "tls"); assertWireValues(SetupApiContract.ExportFormat.values(), "yaml", "env", "kubernetes_secret"); assertWireValues(SetupApiContract.SetupWarningCode.values(), "external_apply_required", "restart_required", - "server_otlp_plaintext", "mail_security_none", "h2_non_production"); + "public_address_plaintext", "mail_security_none", "h2_non_production"); } @Test @@ -93,12 +93,12 @@ class SetupApiContractTest { "restartRequired"); assertComponents(SetupApiContract.TelemetryStoreSummary.class, "kind", "configured", "source", "restartRequired"); - assertComponents(SetupApiContract.OptionalConfigurationSummary.class, "serverOtlpHttpConfigured", - "serverOtlpGrpcConfigured", "retentionConfigured", "mailConfigured"); + assertComponents(SetupApiContract.OptionalConfigurationSummary.class, "publicBaseUrlConfigured", + "serverOtlpHttpConfigured", "serverOtlpGrpcConfigured", "retentionConfigured", "mailConfigured"); assertComponents(SetupApiContract.UnlockRequest.class, "code"); assertComponents(SetupApiContract.UnlockResponse.class, "access", "expiresAt"); assertComponents(SetupApiContract.ValidateRequest.class, "section", "managementDatabase", "telemetryStore", - "serverInstrumentation", "mail"); + "publicAccess", "mail"); assertComponents(SetupApiContract.TelemetryStoreConfiguration.class, "kind", "grpcEndpoints", "httpEndpoint", "database", "username", "password"); assertComponents(SetupApiContract.ValidationResponse.class, "valid", "observedAt", "errorCode", "warnings"); @@ -110,10 +110,13 @@ class SetupApiContractTest { "startedAt", "completedAt", "errorCode", "nextPollAfterMillis", "exportAvailable"); assertComponents(SetupApiContract.AdministratorRequest.class, "username", "password"); assertComponents(SetupApiContract.AdministratorResponse.class, "username", "phase"); - assertComponents(SetupApiContract.OptionsRequest.class, "serverInstrumentation", "retention", "mail"); + assertComponents(SetupApiContract.OptionsRequest.class, "publicAccess", "retention", "mail"); + assertComponents(SetupApiContract.PublicAccessConfiguration.class, "publicBaseUrl", + "serverOtlpHttpEndpoint", "serverOtlpGrpcEndpoint"); assertComponents(SetupApiContract.RetentionConfiguration.class, "days"); - assertComponents(SetupApiContract.OptionsResponse.class, "serverOtlpHttpConfigured", - "serverOtlpGrpcConfigured", "retentionConfigured", "mailConfigured", "phase"); + assertComponents(SetupApiContract.OptionsResponse.class, "publicBaseUrlConfigured", + "serverOtlpHttpConfigured", "serverOtlpGrpcConfigured", "retentionConfigured", "mailConfigured", + "phase"); assertComponents(SetupApiContract.ExportRequest.class, "format", "configuration"); assertComponents(SetupApiContract.ExportResponse.class, "fileName", "mediaType"); assertComponents(SetupApiContract.CompleteRequest.class, "expectedPhase", "acknowledgedWarnings"); @@ -164,6 +167,20 @@ class SetupApiContractTest { assertEquals(SECRET, decoded.code()); } + @Test + void publicAccessInputNeverRendersAddressBodies() { + PublicAccessConfiguration configuration = new PublicAccessConfiguration( + "https://user:" + SECRET + "@hertzbeat.example.test", + "https://collector.example.test:4318?token=" + SECRET, + "https://collector.example.test:4317"); + + assertEquals("PublicAccessConfiguration[publicBaseUrlProvided=true, " + + "serverOtlpHttpEndpointProvided=true, serverOtlpGrpcEndpointProvided=true]", + configuration.toString()); + assertFalse(configuration.toString().contains(SECRET)); + assertFalse(configuration.toString().contains("example.test")); + } + @Test void validateRequestRequiresExactlyOneMatchingSection() { MetadataDatabaseConfiguration metadata = new MetadataDatabaseConfiguration( @@ -174,7 +191,7 @@ class SetupApiContractTest { () -> new ValidateRequest(ValidationSection.METADATA_DATABASE, null, null, null, null)); assertThrows(IllegalArgumentException.class, () -> new ValidateRequest( ValidationSection.METADATA_DATABASE, metadata, null, - new ServerInstrumentationConfiguration("http://localhost:4318", null), null)); + new PublicAccessConfiguration(null, "http://localhost:4318", null), null)); assertThrows(IllegalArgumentException.class, () -> new ValidateRequest( ValidationSection.MAIL, metadata, null, null, null)); } @@ -202,7 +219,7 @@ class SetupApiContractTest { "config_read_only", "config_write_failed", "config_recovery_required", "metadata_connection_failed", "metadata_kind_unsupported", "metadata_schema_mismatch", "metadata_insufficient_privileges", "telemetry_connection_failed", - "server_instrumentation_invalid", "mail_connection_failed", "administrator_already_configured", + "public_address_invalid", "mail_connection_failed", "administrator_already_configured", "administrator_username_invalid", "operation_not_found", "operation_conflict", "migration_source_unsupported", "migration_target_not_empty", "migration_multi_node_unsupported", "migration_copy_failed", "migration_verification_failed", "migration_activation_failed", @@ -224,7 +241,7 @@ class SetupApiContractTest { new SetupApiContract.TelemetryStoreSummary( TelemetryStoreKind.GREPTIME, false, ConfigSource.BUILT_IN_DEFAULT, false), false, - new SetupApiContract.OptionalConfigurationSummary(false, false, false, false)); + new SetupApiContract.OptionalConfigurationSummary(false, false, false, false, false)); String json = objectMapper.writeValueAsString(response); assertFalse(json.contains("jdbc")); assertFalse(json.contains("username")); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupControllerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupControllerTest.java index 6ebd1ac553..b3e3294e8e 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupControllerTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupControllerTest.java @@ -90,7 +90,7 @@ class SetupControllerTest { ConfigSource.BUILT_IN_DEFAULT, false), new TelemetryStoreSummary(TelemetryStoreKind.GREPTIME, false, ConfigSource.BUILT_IN_DEFAULT, false), - false, new OptionalConfigurationSummary(false, false, false, false))); + false, new OptionalConfigurationSummary(false, false, false, false, false))); mvc.perform(get(SetupApiContract.STATUS_PATH)) .andExpect(status().isOk()) diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupStatusProjectionFactoryTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupStatusProjectionFactoryTest.java index 33b74edb11..8013ad0597 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupStatusProjectionFactoryTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupStatusProjectionFactoryTest.java @@ -72,6 +72,25 @@ class SetupStatusProjectionFactoryTest { assertThat(projection.optional().serverOtlpHttpConfigured()).isFalse(); } + @Test + void invalidExternalAddressesAreNotReportedAsConfigured() { + StandardEnvironment environment = new StandardEnvironment(); + environment.getPropertySources().replace( + StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME, + new MapPropertySource(StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME, Map.of( + "spring.jpa.database", "H2", + "warehouse.store.greptime.enabled", "true", + "hertzbeat.setup.public-base-url", "http://0.0.0.0:1157", + "hertzbeat.instrumentation.server.otlp-http-endpoint", "http://collector.example.test:70000"))); + var inspection = new ManagedActiveConfigurationInspector.Inspection( + ManagedActiveConfigurationInspector.State.ABSENT, Map.of(), Map.of()); + + var projection = new SetupStatusProjectionFactory().create(environment, inspection); + + assertThat(projection.optional().publicBaseUrlConfigured()).isFalse(); + assertThat(projection.optional().serverOtlpHttpConfigured()).isFalse(); + } + @Test void restartProjectionUsesEffectiveSourceAndRehydratesSafeManagedOptions() { StandardEnvironment environment = new StandardEnvironment(); @@ -83,6 +102,7 @@ class SetupStatusProjectionFactoryTest { ManagedActiveConfigurationInspector.MANAGED_APPLICATION_SOURCE, Map.of("spring.jpa.database", "H2", "warehouse.store.greptime.enabled", "true", + "hertzbeat.setup.public-base-url", "http://hertzbeat.example.test", "hertzbeat.instrumentation.server.otlp-http-endpoint", "http://localhost:4318", "warehouse.store.greptime.expire-time", "30d", "spring.mail.host", "mail.example.test", @@ -95,11 +115,12 @@ class SetupStatusProjectionFactoryTest { assertThat(projection.managementDatabase().kind()).isEqualTo(MetadataDatabaseKind.POSTGRESQL); assertThat(projection.managementDatabase().source()).isEqualTo(ConfigSource.SYSTEM_PROPERTY); + assertThat(projection.optional().publicBaseUrlConfigured()).isTrue(); assertThat(projection.optional().serverOtlpHttpConfigured()).isTrue(); assertThat(projection.optional().retentionConfigured()).isTrue(); assertThat(projection.optional().mailConfigured()).isTrue(); assertThat(projection.warnings()).containsExactlyInAnyOrder( - SetupWarningCode.SERVER_OTLP_PLAINTEXT, SetupWarningCode.MAIL_SECURITY_NONE); + SetupWarningCode.PUBLIC_ADDRESS_PLAINTEXT, SetupWarningCode.MAIL_SECURITY_NONE); } @Test diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionalConfigurationPersistenceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionalConfigurationPersistenceTest.java index e9e031ced2..84bb49ae96 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionalConfigurationPersistenceTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedOptionalConfigurationPersistenceTest.java @@ -35,7 +35,8 @@ class ManagedOptionalConfigurationPersistenceTest { ManagedConfigurationTransaction transaction = new ManagedConfigurationTransaction(root); assertThat(transaction.apply(required())).isEqualTo(ManagedConfigurationTransaction.Outcome.APPLIED); ManagedOptionalConfiguration options = new ManagedOptionalConfiguration( - Optional.of(new ManagedOptionalConfiguration.ServerInstrumentationSettings( + Optional.of(new ManagedOptionalConfiguration.PublicAccessSettings( + Optional.of("https://hertzbeat.example"), Optional.of("https://hertzbeat.example/otlp"), Optional.of("https://hertzbeat.example:4317"))), Optional.of(new ManagedOptionalConfiguration.RetentionSettings(30)), @@ -53,6 +54,7 @@ class ManagedOptionalConfigurationPersistenceTest { assertThat(secrets.mailPassword()).get().isEqualTo(SecretValue.of("mail-secret")); var properties = ApplicationConfigDocumentCodec.springProperties(application); assertThat(properties) + .containsEntry("hertzbeat.setup.public-base-url", "https://hertzbeat.example") .containsEntry("hertzbeat.instrumentation.server.otlp-http-endpoint", "https://hertzbeat.example/otlp") .containsEntry("hertzbeat.instrumentation.server.otlp-grpc-endpoint", @@ -63,9 +65,7 @@ class ManagedOptionalConfigurationPersistenceTest { .containsEntry("spring.mail.properties.mail.smtp.ssl.enable", "true") .containsEntry("spring.mail.properties.mail.smtp.starttls.enable", "false") .containsEntry("hertzbeat.mail.from-address", "alerts@example.test") - .doesNotContainKeys("hertzbeat.setup.public-base-url", "hertzbeat.setup.retention.metrics-days", - "hertzbeat.setup.retention.logs-days", "hertzbeat.setup.retention.traces-days", - "hertzbeat.setup.mail.security"); + .doesNotContainKey("hertzbeat.setup.mail.security"); assertThat(properties.toString()).doesNotContain("mail-secret"); } @@ -73,8 +73,8 @@ class ManagedOptionalConfigurationPersistenceTest { void rejectsServerEndpointsWithoutCompleteInternalProfileSettings() throws Exception { ManagedApplicationConfig application = required().application(); ManagedOptionalConfiguration options = new ManagedOptionalConfiguration( - Optional.of(new ManagedOptionalConfiguration.ServerInstrumentationSettings( - Optional.of("https://hertzbeat.example/otlp"), Optional.empty())), + Optional.of(new ManagedOptionalConfiguration.PublicAccessSettings( + Optional.empty(), Optional.of("https://hertzbeat.example/otlp"), Optional.empty())), Optional.empty(), Optional.empty()); application = new ManagedApplicationConfig( application.metadataDatabase(), application.telemetryStore(), options); @@ -90,16 +90,38 @@ class ManagedOptionalConfigurationPersistenceTest { } @Test - void rejectsBlankManagedServerEndpoint() { - assertThatThrownBy(() -> new ManagedOptionalConfiguration.ServerInstrumentationSettings( - Optional.of(" "), Optional.empty())) + void publicBaseUrlRoundTripsWithoutInventingServerEndpoints() throws Exception { + ManagedApplicationConfig application = required().application(); + ManagedOptionalConfiguration options = new ManagedOptionalConfiguration( + Optional.of(new ManagedOptionalConfiguration.PublicAccessSettings( + Optional.of("http://192.168.10.5:1157"), Optional.empty(), Optional.empty())), + Optional.empty(), Optional.empty()); + application = new ManagedApplicationConfig( + application.metadataDatabase(), application.telemetryStore(), options); + ApplicationConfigDocumentCodec codec = new ApplicationConfigDocumentCodec(); + + ManagedApplicationConfig decoded = codec.decode(codec.encode(application, "generation")).value(); + + assertThat(decoded.optional()).isEqualTo(options); + assertThat(ApplicationConfigDocumentCodec.springProperties(decoded)) + .containsEntry("hertzbeat.setup.public-base-url", "http://192.168.10.5:1157") + .doesNotContainKeys("hertzbeat.instrumentation.server.otlp-http-endpoint", + "hertzbeat.instrumentation.server.otlp-grpc-endpoint", + "hertzbeat.instrumentation.server.profile-id", + "hertzbeat.instrumentation.server.authentication"); + } + + @Test + void rejectsBlankManagedPublicAddress() { + assertThatThrownBy(() -> new ManagedOptionalConfiguration.PublicAccessSettings( + Optional.of(" "), Optional.empty(), Optional.empty())) .isInstanceOf(IllegalArgumentException.class); } @Test - void rejectsControlOnlyManagedServerEndpoint() { - assertThatThrownBy(() -> new ManagedOptionalConfiguration.ServerInstrumentationSettings( - Optional.of("\u0000"), Optional.empty())) + void rejectsControlOnlyManagedPublicAddress() { + assertThatThrownBy(() -> new ManagedOptionalConfiguration.PublicAccessSettings( + Optional.of("\u0000"), Optional.empty(), Optional.empty())) .isInstanceOf(IllegalArgumentException.class); } @@ -107,8 +129,8 @@ class ManagedOptionalConfigurationPersistenceTest { void rejectsBlankServerEndpointInManagedDocument() throws Exception { ManagedApplicationConfig application = required().application(); ManagedOptionalConfiguration options = new ManagedOptionalConfiguration( - Optional.of(new ManagedOptionalConfiguration.ServerInstrumentationSettings( - Optional.of("https://hertzbeat.example/otlp"), Optional.empty())), + Optional.of(new ManagedOptionalConfiguration.PublicAccessSettings( + Optional.empty(), Optional.of("https://hertzbeat.example/otlp"), Optional.empty())), Optional.empty(), Optional.empty()); application = new ManagedApplicationConfig( application.metadataDatabase(), application.telemetryStore(), options); @@ -124,6 +146,27 @@ class ManagedOptionalConfigurationPersistenceTest { .isInstanceOf(ManagedDocumentCodec.DocumentException.class); } + @Test + void rejectsInvalidPublicAddressInManagedDocument() throws Exception { + ManagedApplicationConfig application = required().application(); + ManagedOptionalConfiguration options = new ManagedOptionalConfiguration( + Optional.of(new ManagedOptionalConfiguration.PublicAccessSettings( + Optional.of("https://hertzbeat.example.test"), Optional.empty(), Optional.empty())), + Optional.empty(), Optional.empty()); + application = new ManagedApplicationConfig( + application.metadataDatabase(), application.telemetryStore(), options); + ApplicationConfigDocumentCodec codec = new ApplicationConfigDocumentCodec(); + ManagedDocumentCodec.Integrity.VerifiedBody encoded = ManagedDocumentCodec.Integrity.extract( + codec.encode(application, "generation")); + String invalid = encoded.content().replace( + "hertzbeat.setup.public-base-url: 'https://hertzbeat.example.test'", + "hertzbeat.setup.public-base-url: 'http://0.0.0.0:1157'"); + + byte[] document = ManagedDocumentCodec.Integrity.envelope(invalid, encoded.generation()); + assertThatThrownBy(() -> codec.decode(document)) + .isInstanceOf(ManagedDocumentCodec.DocumentException.class); + } + private static ManagedConfigurationBundle required() { ManagedApplicationConfig application = new ManagedApplicationConfig( new MetadataDatabaseSettings(MetadataDatabaseKind.H2, "jdbc:h2:./data/hertzbeat", "sa"), diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializerTest.java index 5e23366acf..d3012fc813 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializerTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/UnattendedSetupInitializerTest.java @@ -109,13 +109,13 @@ class UnattendedSetupInitializerTest { MockEnvironment environment = new MockEnvironment() .withProperty(UnattendedSetupInitializer.ENABLED_PROPERTY, "true") .withProperty("hertzbeat.setup.unattended.acknowledged-warnings", - "h2_non_production, server_otlp_plaintext"); + "h2_non_production, public_address_plaintext"); new UnattendedSetupInitializer(workflow, environment, new SetupPasswordFileLoader(), transitions) .initialize(); verify(workflow).complete(java.util.List.of( - SetupWarningCode.H2_NON_PRODUCTION, SetupWarningCode.SERVER_OTLP_PLAINTEXT)); + SetupWarningCode.H2_NON_PRODUCTION, SetupWarningCode.PUBLIC_ADDRESS_PLAINTEXT)); verify(transitions).installationCompleted(); } @@ -180,6 +180,6 @@ class UnattendedSetupInitializerTest { new ManagementDatabaseSummary(MetadataDatabaseKind.H2, true, ConfigSource.UI_MANAGED, false), new TelemetryStoreSummary(TelemetryStoreKind.GREPTIME, true, ConfigSource.UI_MANAGED, false), phase != SetupPhase.ADMINISTRATOR_REQUIRED, - new OptionalConfigurationSummary(false, false, false, false)); + new OptionalConfigurationSummary(false, false, false, false, false)); } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java index 8ed8cba14e..270cc13bdd 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java @@ -46,7 +46,7 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationResp import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsRequest; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ServerInstrumentationConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.PublicAccessConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupAccess; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; @@ -77,13 +77,16 @@ class DefaultSetupWorkflowTest { Optional.of(mock(SetupCompletionCoordinator.class)), mock(SetupOptionsCoordinator.class), Clock.systemUTC(), new SetupMutationSerializer()); OptionsRequest request = new OptionsRequest( - new ServerInstrumentationConfiguration("https://server.example.test:4318", "\u0000"), + new PublicAccessConfiguration("https://hertzbeat.example.test", + "https://server.example.test:4318", "\u0000"), null, null); var response = workflow.configureOptions(request); + assertTrue(response.publicBaseUrlConfigured()); assertTrue(response.serverOtlpHttpConfigured()); assertFalse(response.serverOtlpGrpcConfigured()); + assertTrue(state.status().optional().publicBaseUrlConfigured()); assertTrue(state.status().optional().serverOtlpHttpConfigured()); assertFalse(state.status().optional().serverOtlpGrpcConfigured()); } @@ -156,7 +159,7 @@ class DefaultSetupWorkflowTest { mock(SetupConfigurationCoordinator.class), capability, Optional.of(mock(IdentityInitializationService.class)), Optional.of(completion), mutations); OptionsRequest request = new OptionsRequest( - new ServerInstrumentationConfiguration("http://localhost:4318", null), null, null); + new PublicAccessConfiguration(null, "http://collector.example.test:4318", null), null, null); try (var executor = Executors.newFixedThreadPool(2)) { var optionsResult = executor.submit(() -> workflow.configureOptions(request)); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinatorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinatorTest.java index 921622537d..e0f7b9c6df 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinatorTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinatorTest.java @@ -40,9 +40,9 @@ class HeadlessSetupCoordinatorTest { ManagedConfigCapability capability = mock(ManagedConfigCapability.class); SetupRuntimeState state = new SetupRuntimeState(Clock.systemUTC(), capability, SetupPhase.OPTIONAL_CONFIGURATION, SetupAccess.LOCAL, true, "operator"); - state.optionsConfigured(new OptionalConfigurationSummary(true, false, false, false), + state.optionsConfigured(new OptionalConfigurationSummary(false, true, false, false, false), List.of(SetupWarningCode.H2_NON_PRODUCTION, - SetupWarningCode.SERVER_OTLP_PLAINTEXT)); + SetupWarningCode.PUBLIC_ADDRESS_PLAINTEXT)); SetupCompletionCoordinator completion = mock(SetupCompletionCoordinator.class); SetupTransitionService transitions = new SetupTransitionService(state, mock(SetupRequestValidator.class), mock(SetupConfigurationCoordinator.class), capability, @@ -56,7 +56,7 @@ class HeadlessSetupCoordinatorTest { verifyNoInteractions(completion); coordinator.complete(List.of(SetupWarningCode.H2_NON_PRODUCTION, - SetupWarningCode.SERVER_OTLP_PLAINTEXT)); + SetupWarningCode.PUBLIC_ADDRESS_PLAINTEXT)); verify(completion).completeInstallation(); } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidatorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidatorTest.java index 4bb9425845..5c1cbe8d09 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidatorTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupRequestValidatorTest.java @@ -26,7 +26,7 @@ import java.time.Instant; import java.time.ZoneOffset; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ServerInstrumentationConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.PublicAccessConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidateRequest; @@ -50,60 +50,155 @@ class SetupRequestValidatorTest { } @Test - void serverInstrumentationValidatorProducesStablePlaintextWarning() { - var response = validator.validate(new ValidateRequest(ValidationSection.SERVER_INSTRUMENTATION, - null, null, new ServerInstrumentationConfiguration("http://monitor.example.test", null), null)); + void publicAccessValidatorProducesStablePlaintextWarningForPublicHttp() { + var response = validator.validate(new ValidateRequest(ValidationSection.PUBLIC_ACCESS, + null, null, new PublicAccessConfiguration( + "http://monitor.example.test", null, null), null)); assertTrue(response.valid()); - assertEquals(1, response.warnings().size()); + assertEquals(java.util.List.of(SetupWarningCode.PUBLIC_ADDRESS_PLAINTEXT), response.warnings()); + } + + @Test + void internalHttpAddressIsAllowedWithoutPublicPlaintextWarning() { + var response = validator.validate(new ValidateRequest(ValidationSection.PUBLIC_ACCESS, + null, null, new PublicAccessConfiguration("http://192.168.10.5:1157", null, null), null)); + + assertTrue(response.valid()); + assertTrue(response.warnings().isEmpty()); + } + + @Test + void internalIpv6HttpAddressesDoNotProducePublicPlaintextWarning() { + for (String address : java.util.List.of( + "http://[::1]:1157", "http://[fd00::1]:1157", + "http://[::ffff:192.168.10.5]:1157", "http://[::ffff:127.0.0.1]:1157", + "http://[0:0:0:0::ffff:192.168.10.5]:1157", "http://[::ffff:c0a8:0a05]:1157")) { + var response = validator.validate(new ValidateRequest(ValidationSection.PUBLIC_ACCESS, + null, null, new PublicAccessConfiguration(address, null, null), null)); + + assertTrue(response.valid()); + assertTrue(response.warnings().isEmpty()); + } + } + + @Test + void publicBaseUrlMustBeAnExplicitAbsoluteHttpOrHttpsAddress() { + var relative = validator.validate(new ValidateRequest(ValidationSection.PUBLIC_ACCESS, + null, null, new PublicAccessConfiguration("/from-browser-origin", null, null), null)); + var https = validator.validate(new ValidateRequest(ValidationSection.PUBLIC_ACCESS, + null, null, new PublicAccessConfiguration("https://hertzbeat.example.test", null, null), null)); + + assertFalse(relative.valid()); + assertEquals(SetupErrorCode.PUBLIC_ADDRESS_INVALID, relative.errorCode()); + assertTrue(https.valid()); + assertTrue(https.warnings().isEmpty()); } @Test void serverGrpcEndpointMustBeAnExplicitHttpUrl() { - var response = validator.validate(new ValidateRequest(ValidationSection.SERVER_INSTRUMENTATION, - null, null, new ServerInstrumentationConfiguration(null, "collector.example.test:4317"), null)); + var response = validator.validate(new ValidateRequest(ValidationSection.PUBLIC_ACCESS, + null, null, new PublicAccessConfiguration(null, null, "collector.example.test:4317"), null)); assertFalse(response.valid()); - assertEquals(SetupErrorCode.SERVER_INSTRUMENTATION_INVALID, response.errorCode()); + assertEquals(SetupErrorCode.PUBLIC_ADDRESS_INVALID, response.errorCode()); + } + + @Test + void serverEndpointsRejectPortsOutsideTheTcpRange() { + for (PublicAccessConfiguration configuration : java.util.List.of( + new PublicAccessConfiguration("http://hertzbeat.example.test:0", null, null), + new PublicAccessConfiguration("http://hertzbeat.example.test:70000", null, null), + new PublicAccessConfiguration(null, "http://collector.example.test:0", null), + new PublicAccessConfiguration(null, "http://collector.example.test:70000", null), + new PublicAccessConfiguration(null, null, "http://collector.example.test:0"), + new PublicAccessConfiguration(null, null, "http://collector.example.test:70000"))) { + var response = validator.validate(new ValidateRequest(ValidationSection.PUBLIC_ACCESS, + null, null, configuration, null)); + + assertFalse(response.valid()); + assertEquals(SetupErrorCode.PUBLIC_ADDRESS_INVALID, response.errorCode()); + } + } + + @Test + void advertisedAddressesRejectWildcardHosts() { + for (PublicAccessConfiguration configuration : java.util.List.of( + new PublicAccessConfiguration("http://0.0.0.0:1157", null, null), + new PublicAccessConfiguration("http://[::]:1157", null, null), + new PublicAccessConfiguration("http://[::ffff:0.0.0.0]:1157", null, null), + new PublicAccessConfiguration("http://[::ffff:0:0]:1157", null, null), + new PublicAccessConfiguration(null, "http://0.0.0.0:4318", null), + new PublicAccessConfiguration(null, null, "http://[0:0:0:0:0:0:0:0]:4317"))) { + var response = validator.validate(new ValidateRequest(ValidationSection.PUBLIC_ACCESS, + null, null, configuration, null)); + + assertFalse(response.valid()); + assertEquals(SetupErrorCode.PUBLIC_ADDRESS_INVALID, response.errorCode()); + } + } + + @Test + void advertisedIpv6AddressesRejectZoneIdentifiers() { + for (PublicAccessConfiguration configuration : java.util.List.of( + new PublicAccessConfiguration("http://[::%25eth0]:4318", null, null), + new PublicAccessConfiguration("http://[::%eth0]:4318", null, null), + new PublicAccessConfiguration(null, "http://[fe80::1%25eth0]:4318", null), + new PublicAccessConfiguration(null, "http://[fe80::1%eth0]:4318", null))) { + var response = validator.validate(new ValidateRequest(ValidationSection.PUBLIC_ACCESS, + null, null, configuration, null)); + + assertFalse(response.valid()); + assertEquals(SetupErrorCode.PUBLIC_ADDRESS_INVALID, response.errorCode()); + } + } + + @Test + void publicIpv4MappedIpv6AddressStillProducesPlaintextWarning() { + var response = validator.validate(new ValidateRequest(ValidationSection.PUBLIC_ACCESS, + null, null, new PublicAccessConfiguration("http://[::ffff:0808:0808]:1157", null, null), null)); + + assertTrue(response.valid()); + assertEquals(java.util.List.of(SetupWarningCode.PUBLIC_ADDRESS_PLAINTEXT), response.warnings()); } @Test void serverEndpointRejectsUrlCredentialsAndQuery() { - var response = validator.validate(new ValidateRequest(ValidationSection.SERVER_INSTRUMENTATION, - null, null, new ServerInstrumentationConfiguration( + var response = validator.validate(new ValidateRequest(ValidationSection.PUBLIC_ACCESS, + null, null, new PublicAccessConfiguration(null, "https://user:secret@collector.example.test:4318?token=secret", null), null)); assertFalse(response.valid()); - assertEquals(SetupErrorCode.SERVER_INSTRUMENTATION_INVALID, response.errorCode()); + assertEquals(SetupErrorCode.PUBLIC_ADDRESS_INVALID, response.errorCode()); } @Test void grpcOnlyPlaintextEndpointProducesWarning() { - var response = validator.validate(new ValidateRequest(ValidationSection.SERVER_INSTRUMENTATION, - null, null, new ServerInstrumentationConfiguration(null, "http://collector.example.test:4317"), - null)); + var response = validator.validate(new ValidateRequest(ValidationSection.PUBLIC_ACCESS, + null, null, new PublicAccessConfiguration( + null, null, "http://collector.example.test:4317"), null)); assertTrue(response.valid()); - assertEquals(java.util.List.of(SetupWarningCode.SERVER_OTLP_PLAINTEXT), response.warnings()); + assertEquals(java.util.List.of(SetupWarningCode.PUBLIC_ADDRESS_PLAINTEXT), response.warnings()); } @Test - void serverInstrumentationSectionRequiresAtLeastOneEndpoint() { - var response = validator.validate(new ValidateRequest(ValidationSection.SERVER_INSTRUMENTATION, - null, null, new ServerInstrumentationConfiguration(" ", null), null)); + void publicAccessSectionRequiresAtLeastOneExplicitAddress() { + var response = validator.validate(new ValidateRequest(ValidationSection.PUBLIC_ACCESS, + null, null, new PublicAccessConfiguration(" ", null, null), null)); assertFalse(response.valid()); - assertEquals(SetupErrorCode.SERVER_INSTRUMENTATION_INVALID, response.errorCode()); + assertEquals(SetupErrorCode.PUBLIC_ADDRESS_INVALID, response.errorCode()); } @Test void endpointWhitespaceIsNormalizedBeforeValidationAndWarnings() { - var response = validator.validate(new ValidateRequest(ValidationSection.SERVER_INSTRUMENTATION, - null, null, new ServerInstrumentationConfiguration( + var response = validator.validate(new ValidateRequest(ValidationSection.PUBLIC_ACCESS, + null, null, new PublicAccessConfiguration(null, " https://collector.example.test:4318/otlp ", " http://collector.example.test:4317 "), null)); assertTrue(response.valid()); - assertEquals(java.util.List.of(SetupWarningCode.SERVER_OTLP_PLAINTEXT), response.warnings()); + assertEquals(java.util.List.of(SetupWarningCode.PUBLIC_ADDRESS_PLAINTEXT), response.warnings()); } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionServiceTest.java index e5e0a0c393..b83cb6d6b6 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionServiceTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionServiceTest.java @@ -36,7 +36,7 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseC import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionalConfigurationSummary; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsRequest; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ServerInstrumentationConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.PublicAccessConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupAccess; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; @@ -226,10 +226,10 @@ class SetupTransitionServiceTest { } @Test - void serverInstrumentationValidationFailureDoesNotPersistOrPublishState() { + void publicAccessValidationFailureDoesNotPersistOrPublishState() { assertOptionsValidationFailure(new OptionsRequest( - new ServerInstrumentationConfiguration("not-an-endpoint", null), null, null), - ValidationSection.SERVER_INSTRUMENTATION, SetupErrorCode.SERVER_INSTRUMENTATION_INVALID); + new PublicAccessConfiguration("not-an-address", null, null), null, null), + ValidationSection.PUBLIC_ACCESS, SetupErrorCode.PUBLIC_ADDRESS_INVALID); } @Test @@ -270,7 +270,7 @@ class SetupTransitionServiceTest { ConfigSource.UI_MANAGED, false), new TelemetryStoreSummary(TelemetryStoreKind.GREPTIME, true, ConfigSource.UI_MANAGED, false), - new OptionalConfigurationSummary(false, false, false, false), List.of()); + new OptionalConfigurationSummary(false, false, false, false, false), List.of()); SetupRuntimeState state = new SetupRuntimeState(CLOCK, capability, SetupPhase.OPTIONAL_CONFIGURATION, SetupAccess.LOCAL, true, "operator", projection); SetupRequestValidator validator = mock(SetupRequestValidator.class); @@ -281,16 +281,18 @@ class SetupTransitionServiceTest { mock(SetupConfigurationCoordinator.class), capability, options, Optional.empty(), Optional.empty()); OptionsRequest request = new OptionsRequest( - new ServerInstrumentationConfiguration(" ", "https://server.example.test:4317"), null, + new PublicAccessConfiguration("https://hertzbeat.example.test", " ", + "https://server.example.test:4317"), null, new MailConfiguration("mail.example.test", 25, MailSecurity.NONE, null, null, "alerts@example.test")); var response = transitions.configureOptions(request); + assertThat(response.publicBaseUrlConfigured()).isTrue(); assertThat(response.serverOtlpHttpConfigured()).isFalse(); assertThat(response.serverOtlpGrpcConfigured()).isTrue(); assertThat(state.status().optional()).isEqualTo( - new OptionalConfigurationSummary(false, true, false, true)); + new OptionalConfigurationSummary(true, false, true, false, true)); assertThat(state.pendingWarnings()).containsExactly(SetupWarningCode.MAIL_SECURITY_NONE); verify(options).persist(request); } @@ -382,7 +384,7 @@ class SetupTransitionServiceTest { private static OptionsRequest optionsRequest() { return new OptionsRequest( - new ServerInstrumentationConfiguration("https://server.example.test:4318", null), null, null); + new PublicAccessConfiguration(null, "https://server.example.test:4318", null), null, null); } private static HeadlessSetupWorkflow.RequiredConfiguration headlessConfiguration(SecretValue password) { diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupWarningPolicyTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupWarningPolicyTest.java index c79687f94c..fe530d6a52 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupWarningPolicyTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupWarningPolicyTest.java @@ -23,7 +23,7 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailConfiguration import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsRequest; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ServerInstrumentationConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.PublicAccessConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; import org.junit.jupiter.api.Test; @@ -31,19 +31,20 @@ class SetupWarningPolicyTest { @Test void liveAndRestartInputsProduceTheSameWarnings() { var options = new OptionsRequest( - new ServerInstrumentationConfiguration("http://localhost:4318", null), null, + new PublicAccessConfiguration("http://localhost:1157", "http://localhost:4318", null), null, new MailConfiguration("localhost", 25, MailSecurity.NONE, null, null, "ops@example.test")); assertThat(SetupWarningPolicy.INSTANCE.evaluate(MetadataDatabaseKind.H2, options)) .containsExactlyElementsOf(SetupWarningPolicy.INSTANCE.evaluate( - MetadataDatabaseKind.H2, "http://localhost:4318", null, MailSecurity.NONE)); + MetadataDatabaseKind.H2, "http://localhost:1157", "http://localhost:4318", null, + MailSecurity.NONE)); } @Test void whitespaceWrappedGrpcEndpointStillProducesPlaintextWarning() { var options = new OptionsRequest( - new ServerInstrumentationConfiguration(null, " http://localhost:4317 "), null, null); + new PublicAccessConfiguration(null, null, " http://collector.example.test:4317 "), null, null); assertThat(SetupWarningPolicy.INSTANCE.evaluate(MetadataDatabaseKind.MYSQL, options)) - .containsExactly(SetupWarningCode.SERVER_OTLP_PLAINTEXT); + .containsExactly(SetupWarningCode.PUBLIC_ADDRESS_PLAINTEXT); } } From a6cadbc7b7e2bc41e5a4686113f00780a4ec20f6 Mon Sep 17 00:00:00 2001 From: Logic Date: Sun, 9 Aug 2026 12:30:46 +0800 Subject: [PATCH 27/71] Refine setup local file handling --- .../setup/api/SetupApiConfiguration.java | 13 +- .../setup/api/SetupRuntimeStateFactory.java | 1 + .../setup/config/NioManagedFilePublisher.java | 34 ++- .../InstallationConvergenceService.java | 8 +- .../LocalInstallationFingerprintStore.java | 24 ++- .../security/OwnerOnlyFilePermissions.java | 149 +++++++++++++ .../setup/security/RemoteSetupUnlock.java | 21 +- .../security/SecureFileChannelReader.java | 56 +++++ .../setup/security/SecureSetupFile.java | 196 +++++++++++++++--- .../unattended/SetupPasswordFileLoader.java | 30 +-- .../setup/unattended/Utf8SecretDecoder.java | 46 ++++ .../config/NioManagedFilePublisherTest.java | 34 +++ .../InstallationConvergenceServiceTest.java | 13 +- .../InstallationPersistenceTest.java | 25 ++- .../OwnerOnlyFilePermissionsTest.java | 73 +++++++ .../setup/security/RemoteSetupUnlockTest.java | 83 +++++++- .../security/SecureFileChannelReaderTest.java | 120 +++++++++++ .../setup/security/SecureSetupFileTest.java | 100 +++++++++ .../security/SetupHttpUnlockServiceTest.java | 12 +- .../SetupPasswordFileLoaderTest.java | 39 +++- .../unattended/Utf8SecretDecoderTest.java | 41 ++++ 21 files changed, 996 insertions(+), 122 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/OwnerOnlyFilePermissions.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureFileChannelReader.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/unattended/Utf8SecretDecoder.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/OwnerOnlyFilePermissionsTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SecureFileChannelReaderTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/Utf8SecretDecoderTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java index 3b18ec0907..2f385540d0 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java @@ -121,10 +121,12 @@ public class SetupApiConfiguration { @Bean(destroyMethod = "close") public SetupHttpUnlockService setupHttpUnlockService( Environment environment, SetupRuntimeState state) throws IOException { - Path codeFile = SetupInstallationPaths.root(environment).resolve("data/config/setup-unlock-code"); + Path installationRoot = SetupInstallationPaths.root(environment); + Path codeFile = installationRoot.resolve("data/config/setup-unlock-code"); InetAddress bindAddress = bindAddress(environment.getProperty("server.address")); Clock clock = Clock.systemUTC(); - return new SetupHttpUnlockService(new RemoteSetupUnlock(codeFile, clock, new SecureRandom()), + return new SetupHttpUnlockService(new RemoteSetupUnlock( + installationRoot, codeFile, clock, new SecureRandom()), bindAddress, state, clock); } @@ -186,10 +188,11 @@ public class SetupApiConfiguration { private Optional completion( Environment environment, Optional installations) { - Path fingerprint = SetupInstallationPaths.root(environment) - .resolve("data/config/.installation-fingerprint"); + Path installationRoot = SetupInstallationPaths.root(environment); + Path fingerprint = installationRoot.resolve("data/config/.installation-fingerprint"); return installations.map(service -> new SetupCompletionCoordinator( - new LocalInstallationFingerprintStore(fingerprint, new SecureRandom()), service)); + new LocalInstallationFingerprintStore( + installationRoot, fingerprint, new SecureRandom()), service)); } static InetAddress bindAddress(String configured) { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupRuntimeStateFactory.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupRuntimeStateFactory.java index e765bca37a..8dee232ec7 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupRuntimeStateFactory.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupRuntimeStateFactory.java @@ -34,6 +34,7 @@ final class SetupRuntimeStateFactory { var administrator = accounts.flatMap(DatabaseAccountRepository::findByBootstrapSlotIsNotNull); Optional installationMode = installations.map(repository -> new InstallationConvergenceService(repository, + root, root.resolve("data/config/.installation-fingerprint")).classify()); SetupPhase phase = phase(gate.mode(), inspection.state(), administrator.isPresent(), installationMode); return new SetupRuntimeState(Clock.systemUTC(), capability, phase, diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/NioManagedFilePublisher.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/NioManagedFilePublisher.java index 75edf8fe6a..b56f641fe9 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/NioManagedFilePublisher.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/NioManagedFilePublisher.java @@ -24,15 +24,12 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.PosixFilePermission; -import java.util.Set; +import java.util.UUID; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; /** Durable temp-write, file-fsync, replace, and directory-fsync publication. */ final class NioManagedFilePublisher implements ManagedFileIo.Publisher { - private static final Set OWNER_ONLY = Set.of( - PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE); - private final ManagedFileIo.Operations operations; NioManagedFilePublisher() { @@ -47,18 +44,14 @@ final class NioManagedFilePublisher implements ManagedFileIo.Publisher { public void publish(Path target, byte[] content, boolean ownerOnly) throws IOException { Path directory = target.toAbsolutePath().getParent(); Files.createDirectories(directory); - Path temporary = Files.createTempFile(directory, ".managed-config-", ".tmp"); + Path temporary = ownerOnly + ? directory.resolve(".managed-config-" + UUID.randomUUID() + ".tmp") + : Files.createTempFile(directory, ".managed-config-", ".tmp"); try { if (ownerOnly) { - setOwnerOnlyWhenSupported(temporary); - } - try (FileChannel channel = FileChannel.open( - temporary, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) { - ByteBuffer buffer = ByteBuffer.wrap(content); - while (buffer.hasRemaining()) { - channel.write(buffer); - } - channel.force(true); + SecureSetupFile.create(directory, temporary, content); + } else { + writeAndForce(temporary, content); } replaceAndForce(temporary, target); } finally { @@ -78,9 +71,14 @@ final class NioManagedFilePublisher implements ManagedFileIo.Publisher { operations.forceDirectory(target.toAbsolutePath().getParent()); } - private static void setOwnerOnlyWhenSupported(Path path) throws IOException { - if (Files.getFileStore(path).supportsFileAttributeView("posix")) { - Files.setPosixFilePermissions(path, OWNER_ONLY); + private static void writeAndForce(Path target, byte[] content) throws IOException { + try (FileChannel channel = FileChannel.open( + target, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) { + ByteBuffer buffer = ByteBuffer.wrap(content); + while (buffer.hasRemaining()) { + channel.write(buffer); + } + channel.force(true); } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationConvergenceService.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationConvergenceService.java index 1bfcbb366a..50bd4d960a 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationConvergenceService.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/InstallationConvergenceService.java @@ -27,17 +27,21 @@ import java.util.Optional; /** Compares the durable database marker with the owner-only local fingerprint. */ public final class InstallationConvergenceService { private final InstallationRecordRepository records; + private final Path installationRoot; private final Path fingerprintPath; - public InstallationConvergenceService(InstallationRecordRepository records, Path fingerprintPath) { + public InstallationConvergenceService( + InstallationRecordRepository records, Path installationRoot, Path fingerprintPath) { this.records = records; + this.installationRoot = installationRoot.toAbsolutePath().normalize(); this.fingerprintPath = fingerprintPath.toAbsolutePath().normalize(); } public InstallationMode classify() { try { Optional fingerprint = - new LocalInstallationFingerprintStore(fingerprintPath, new SecureRandom()).read(); + new LocalInstallationFingerprintStore( + installationRoot, fingerprintPath, new SecureRandom()).read(); if (fingerprint.isEmpty() && Files.exists(fingerprintPath, LinkOption.NOFOLLOW_LINKS)) { return InstallationMode.RECOVERY; } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/LocalInstallationFingerprintStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/LocalInstallationFingerprintStore.java index 6d03d53281..2a7b7e6f70 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/LocalInstallationFingerprintStore.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/installation/LocalInstallationFingerprintStore.java @@ -19,7 +19,6 @@ package org.apache.hertzbeat.manager.setup.installation; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.nio.file.Path; import java.security.SecureRandom; import java.util.Arrays; @@ -29,19 +28,32 @@ import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; /** Owner-only local installation identity store. */ public final class LocalInstallationFingerprintStore { + private static final int MAX_FINGERPRINT_BYTES = 128; + private final Path installationRoot; private final Path path; private final SecureRandom random; - public LocalInstallationFingerprintStore(Path path, SecureRandom random) { + public LocalInstallationFingerprintStore(Path installationRoot, Path path, SecureRandom random) { + this.installationRoot = installationRoot.toAbsolutePath().normalize(); this.path = path.toAbsolutePath().normalize(); this.random = random; } public Optional read() throws IOException { - if (!SecureSetupFile.isOwnerOnlyRegularFile(path)) { + if (!SecureSetupFile.existsInsideRootWithoutLinks(installationRoot, path)) { return Optional.empty(); } - return Optional.of(new InstallationFingerprint(Files.readString(path, StandardCharsets.US_ASCII).trim())); + byte[] encoded = null; + try { + encoded = SecureSetupFile.readOwnerOnlyWithoutLinks( + installationRoot, path, MAX_FINGERPRINT_BYTES); + return Optional.of(new InstallationFingerprint( + new String(encoded, StandardCharsets.US_ASCII).trim())); + } finally { + if (encoded != null) { + Arrays.fill(encoded, (byte) 0); + } + } } public InstallationFingerprint create() throws IOException { @@ -49,8 +61,8 @@ public final class LocalInstallationFingerprintStore { random.nextBytes(value); InstallationFingerprint fingerprint = new InstallationFingerprint(HexFormat.of().formatHex(value)); try { - SecureSetupFile.ensureSafeParent(path); - SecureSetupFile.create(path, fingerprint.value().getBytes(StandardCharsets.US_ASCII)); + SecureSetupFile.create(installationRoot, path, + fingerprint.value().getBytes(StandardCharsets.US_ASCII)); return fingerprint; } finally { Arrays.fill(value, (byte) 0); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/OwnerOnlyFilePermissions.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/OwnerOnlyFilePermissions.java new file mode 100644 index 0000000000..1eb4721bc9 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/OwnerOnlyFilePermissions.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.security; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.attribute.AclEntry; +import java.nio.file.attribute.AclEntryPermission; +import java.nio.file.attribute.AclEntryType; +import java.nio.file.attribute.AclFileAttributeView; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFileAttributeView; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.nio.file.attribute.UserPrincipal; +import java.util.List; +import java.util.Set; + +/** Cross-platform owner-only permission policy for setup files. */ +final class OwnerOnlyFilePermissions { + private static final Set POSIX_PERMISSIONS = + PosixFilePermissions.fromString("rw-------"); + private static final Set POSIX_READ_ONLY = + PosixFilePermissions.fromString("r--------"); + private static final Set ACL_PERMISSIONS = Set.of( + AclEntryPermission.READ_DATA, + AclEntryPermission.WRITE_DATA, + AclEntryPermission.APPEND_DATA, + AclEntryPermission.READ_ATTRIBUTES, + AclEntryPermission.READ_ACL, + AclEntryPermission.SYNCHRONIZE); + + private OwnerOnlyFilePermissions() { + } + + static FileAttribute[] creationAttributes(Path parent) throws IOException { + if (Files.getFileStore(parent).supportsFileAttributeView(PosixFileAttributeView.class)) { + return new FileAttribute[] {PosixFilePermissions.asFileAttribute(POSIX_PERMISSIONS)}; + } + if (aclView(parent) != null) { + return new FileAttribute[] {aclCreationAttribute()}; + } + throw new IOException("Owner-only file permissions are unavailable"); + } + + static FileAttribute> aclCreationAttribute() { + return new FileAttribute<>() { + @Override + public String name() { + return "acl:acl"; + } + + @Override + public List value() { + return List.of(); + } + }; + } + + static void enforce(Path target) throws IOException { + PosixFileAttributeView posix = posixView(target); + if (posix != null) { + posix.setPermissions(POSIX_PERMISSIONS); + if (!posix.readAttributes().permissions().equals(POSIX_PERMISSIONS)) { + throw verificationFailure(); + } + return; + } + enforceAcl(target); + } + + static boolean isReadableOwnerOnly(Path target) throws IOException { + PosixFileAttributeView posix = posixView(target); + if (posix != null) { + Set permissions = posix.readAttributes().permissions(); + return permissions.equals(POSIX_READ_ONLY) || permissions.equals(POSIX_PERMISSIONS); + } + AclFileAttributeView acl = aclView(target); + return acl != null && isReadableOwnerOnlyAcl( + acl.getAcl(), Files.getOwner(target, LinkOption.NOFOLLOW_LINKS)); + } + + private static void enforceAcl(Path target) throws IOException { + AclFileAttributeView acl = aclView(target); + if (acl == null) { + throw new IOException("Owner-only file permissions are unavailable"); + } + UserPrincipal owner = Files.getOwner(target, LinkOption.NOFOLLOW_LINKS); + acl.setAcl(List.of(ownerEntry(owner))); + if (!hasOwnerOnlyAcl(acl.getAcl(), owner)) { + throw verificationFailure(); + } + } + + private static boolean hasOwnerOnlyAcl(List entries, UserPrincipal owner) { + return entries.size() == 1 + && entries.getFirst().type() == AclEntryType.ALLOW + && entries.getFirst().principal().equals(owner) + && entries.getFirst().permissions().equals(ACL_PERMISSIONS); + } + + private static AclEntry ownerEntry(UserPrincipal owner) { + return AclEntry.newBuilder().setType(AclEntryType.ALLOW) + .setPrincipal(owner).setPermissions(ACL_PERMISSIONS).build(); + } + + static boolean isReadableOwnerOnlyAcl(List entries, UserPrincipal owner) { + boolean ownerCanRead = false; + for (AclEntry entry : entries) { + if (entry.type() != AclEntryType.ALLOW) { + continue; + } + if (!entry.principal().equals(owner)) { + return false; + } + ownerCanRead |= entry.permissions().contains(AclEntryPermission.READ_DATA); + } + return ownerCanRead; + } + + private static PosixFileAttributeView posixView(Path target) { + return Files.getFileAttributeView(target, PosixFileAttributeView.class, LinkOption.NOFOLLOW_LINKS); + } + + private static AclFileAttributeView aclView(Path target) { + return Files.getFileAttributeView(target, AclFileAttributeView.class, LinkOption.NOFOLLOW_LINKS); + } + + private static IOException verificationFailure() { + return new IOException("Owner-only file permissions could not be verified"); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/RemoteSetupUnlock.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/RemoteSetupUnlock.java index 16c76c68a6..073792c010 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/RemoteSetupUnlock.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/RemoteSetupUnlock.java @@ -22,8 +22,6 @@ import java.net.InetAddress; import java.nio.ByteBuffer; import java.nio.CharBuffer; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.LinkOption; import java.nio.file.Path; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -44,6 +42,7 @@ public final class RemoteSetupUnlock { private static final Duration TTL = Duration.ofMinutes(15); private static final int MAX_ATTEMPTS = 5; private static final int MAX_CLIENTS = 1024; + private final Path installationRoot; private final Path codeFile; private final Clock clock; private final SecureRandom random; @@ -52,7 +51,8 @@ public final class RemoteSetupUnlock { private byte[] sessionDigest; private Instant expiresAt; - public RemoteSetupUnlock(Path codeFile, Clock clock, SecureRandom random) { + public RemoteSetupUnlock(Path installationRoot, Path codeFile, Clock clock, SecureRandom random) { + this.installationRoot = installationRoot.toAbsolutePath().normalize(); this.codeFile = codeFile.toAbsolutePath().normalize(); this.clock = clock; this.random = random; @@ -63,7 +63,6 @@ public final class RemoteSetupUnlock { } public synchronized void open() throws IOException { - SecureSetupFile.ensureSafeParent(codeFile); removeStaleCodeFile(); clearDigest(sessionDigest); sessionDigest = null; @@ -76,7 +75,7 @@ public final class RemoteSetupUnlock { clearDigest(codeDigest); codeDigest = digest(encodedCode); expiresAt = clock.instant().plus(TTL); - SecureSetupFile.create(codeFile, encodedCode); + SecureSetupFile.create(installationRoot, codeFile, encodedCode); } catch (IOException | RuntimeException exception) { clearDigest(codeDigest); codeDigest = null; @@ -101,13 +100,7 @@ public final class RemoteSetupUnlock { } private void removeStaleCodeFile() throws IOException { - if (!Files.exists(codeFile, LinkOption.NOFOLLOW_LINKS)) { - return; - } - if (!SecureSetupFile.isOwnerOnlyRegularFile(codeFile)) { - throw new IOException("Existing setup unlock path is not an owner-only regular file"); - } - Files.delete(codeFile); + SecureSetupFile.deleteOwnerOnlyInsideRoot(installationRoot, codeFile); } public synchronized SetupAccessSession redeem(String remoteAddress, SetupUnlockCode supplied) throws IOException { @@ -142,7 +135,7 @@ public final class RemoteSetupUnlock { newSessionDigest = digest(token); // Publish the in-memory session only after the one-time proof is durably unavailable. - Files.deleteIfExists(codeFile); + SecureSetupFile.deleteOwnerOnlyInsideRoot(installationRoot, codeFile); clearDigest(sessionDigest); sessionDigest = newSessionDigest; newSessionDigest = null; @@ -178,7 +171,7 @@ public final class RemoteSetupUnlock { codeDigest = null; sessionDigest = null; attempts.clear(); - Files.deleteIfExists(codeFile); + SecureSetupFile.deleteOwnerOnlyInsideRoot(installationRoot, codeFile); } private static byte[] digest(String value) { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureFileChannelReader.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureFileChannelReader.java new file mode 100644 index 0000000000..31ee647d87 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureFileChannelReader.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.security; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ReadableByteChannel; +import java.util.Arrays; + +/** Reads into caller-owned secret storage and clears it before propagating any failure. */ +final class SecureFileChannelReader { + + private SecureFileChannelReader() { + } + + static void readAndValidate(ReadableByteChannel channel, byte[] destination, Validation validation) + throws IOException { + try { + readFully(channel, destination); + validation.verify(); + } catch (IOException | RuntimeException | Error failure) { + Arrays.fill(destination, (byte) 0); + throw failure; + } + } + + private static void readFully(ReadableByteChannel channel, byte[] destination) throws IOException { + ByteBuffer buffer = ByteBuffer.wrap(destination); + while (buffer.hasRemaining()) { + int read = channel.read(buffer); + if (read <= 0) { + throw new IOException("Setup file changed while it was read"); + } + } + } + + @FunctionalInterface + interface Validation { + void verify() throws IOException; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFile.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFile.java index 555b04dd83..2ca7a5d379 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFile.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFile.java @@ -20,51 +20,191 @@ package org.apache.hertzbeat.manager.setup.security; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; +import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; +import java.nio.file.attribute.FileAttribute; import java.util.Set; -/** Creates a new local secret without following links or exposing permissive content. */ +/** Secure creation, permission enforcement, and bounded reading of local setup files. */ public final class SecureSetupFile { - private static final Set OWNER_READ_WRITE = - PosixFilePermissions.fromString("rw-------"); - private SecureSetupFile() { } - public static void ensureSafeParent(Path target) throws IOException { - Path parent = target.getParent(); - Files.createDirectories(parent); - if (Files.isSymbolicLink(parent)) { - throw new IOException("Setup secret parent must not be a symbolic link"); + /** + * Creates below an operator-controlled root whose descendants cannot be concurrently replaced by untrusted users. + */ + public static void create(Path trustedRoot, Path target, byte[] content) throws IOException { + Path absoluteRoot = absolute(trustedRoot); + Path absoluteTarget = absolute(target); + if (!absoluteTarget.startsWith(absoluteRoot) || absoluteTarget.equals(absoluteRoot)) { + throw new IOException("Setup file resolves outside its trusted root"); + } + createMissingParents(absoluteRoot); + Path resolvedRoot = absoluteRoot.toRealPath(); + Path resolvedTarget = resolvedRoot.resolve(absoluteRoot.relativize(absoluteTarget)); + createSafeDescendants(resolvedRoot, resolvedTarget.getParent()); + if (Files.exists(resolvedTarget, LinkOption.NOFOLLOW_LINKS)) { + throw new FileAlreadyExistsException("Setup file already exists"); + } + FileAttribute[] attributes = OwnerOnlyFilePermissions.creationAttributes(resolvedTarget.getParent()); + boolean created = false; + try { + try (FileChannel channel = FileChannel.open(resolvedTarget, + Set.of(StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE, LinkOption.NOFOLLOW_LINKS), + attributes)) { + created = true; + enforceOwnerOnly(resolvedTarget); + writeAndForce(channel, content); + } + } catch (IOException | RuntimeException failure) { + if (created) { + Files.deleteIfExists(resolvedTarget); + } + throw failure; } } - public static void create(Path target, byte[] content) throws IOException { - Path resolvedTarget = target.getParent().toRealPath().resolve(target.getFileName()); - if (!Files.getFileStore(resolvedTarget.getParent()).supportsFileAttributeView("posix")) { - throw new IOException("Owner-only setup secrets require POSIX file permissions"); - } - try (FileChannel channel = FileChannel.open(resolvedTarget, - Set.of(StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE, LinkOption.NOFOLLOW_LINKS), - PosixFilePermissions.asFileAttribute(OWNER_READ_WRITE))) { - ByteBuffer buffer = ByteBuffer.wrap(content); - while (buffer.hasRemaining()) { - channel.write(buffer); - } - channel.force(true); - } + public static void enforceOwnerOnly(Path target) throws IOException { + OwnerOnlyFilePermissions.enforce(absolute(target)); } public static boolean isOwnerOnlyRegularFile(Path target) throws IOException { - if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) - || !Files.getFileStore(target).supportsFileAttributeView("posix")) { + Path absoluteTarget = absolute(target); + if (!Files.isRegularFile(absoluteTarget, LinkOption.NOFOLLOW_LINKS)) { return false; } - return Files.getPosixFilePermissions(target, LinkOption.NOFOLLOW_LINKS).equals(OWNER_READ_WRITE); + return OwnerOnlyFilePermissions.isReadableOwnerOnly(absoluteTarget); + } + + public static byte[] readOwnerOnly(Path trustedRoot, Path target, int maximumBytes) throws IOException { + return readResolvedOwnerOnly(resolveInsideRoot(trustedRoot, target), maximumBytes); + } + + public static byte[] readOwnerOnlyWithoutLinks(Path trustedRoot, Path target, int maximumBytes) + throws IOException { + return readResolvedOwnerOnly(resolveWithoutLinksInsideRoot(trustedRoot, target), maximumBytes); + } + + public static boolean existsInsideRootWithoutLinks(Path trustedRoot, Path target) throws IOException { + if (Files.isSymbolicLink(absolute(target))) { + return false; + } + return Files.exists(resolveWithoutLinksInsideRoot(trustedRoot, target), LinkOption.NOFOLLOW_LINKS); + } + + public static boolean deleteOwnerOnlyInsideRoot(Path trustedRoot, Path target) throws IOException { + Path resolvedTarget = resolveWithoutLinksInsideRoot(trustedRoot, target); + if (!Files.exists(resolvedTarget, LinkOption.NOFOLLOW_LINKS)) { + return false; + } + if (!isOwnerOnlyRegularFile(resolvedTarget)) { + throw new IOException("Setup file is not an owner-only regular file"); + } + Files.delete(resolvedTarget); + return true; + } + + private static byte[] readResolvedOwnerOnly(Path resolvedTarget, int maximumBytes) throws IOException { + if (!isOwnerOnlyRegularFile(resolvedTarget)) { + throw new IOException("Setup file is not an owner-only regular file"); + } + byte[] content = null; + try { + try (FileChannel channel = FileChannel.open( + resolvedTarget, Set.of(StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS))) { + long size = channel.size(); + if (size <= 0 || size > maximumBytes) { + throw new IOException("Setup file size is invalid"); + } + content = new byte[(int) size]; + SecureFileChannelReader.readAndValidate(channel, content, () -> { + if (channel.size() != size || !isOwnerOnlyRegularFile(resolvedTarget)) { + throw new IOException("Setup file changed while it was read"); + } + }); + } + byte[] transferred = content; + content = null; + return transferred; + } finally { + if (content != null) { + java.util.Arrays.fill(content, (byte) 0); + } + } + } + + static Path resolveInsideRoot(Path trustedRoot, Path target) throws IOException { + Path resolvedRoot = absolute(trustedRoot).toRealPath(); + Path resolvedTarget = absolute(target).toRealPath(); + if (!resolvedTarget.startsWith(resolvedRoot) || resolvedTarget.equals(resolvedRoot)) { + throw new IOException("Setup file resolves outside its trusted root"); + } + return resolvedTarget; + } + + private static Path resolveWithoutLinksInsideRoot(Path trustedRoot, Path target) throws IOException { + Path absoluteRoot = absolute(trustedRoot); + Path absoluteTarget = absolute(target); + if (!absoluteTarget.startsWith(absoluteRoot) || absoluteTarget.equals(absoluteRoot)) { + throw new IOException("Setup file resolves outside its trusted root"); + } + Path current = absoluteRoot.toRealPath(); + for (Path segment : absoluteRoot.relativize(absoluteTarget)) { + current = current.resolve(segment); + if (Files.isSymbolicLink(current)) { + throw new IOException("Setup file path contains a symbolic link"); + } + } + return current; + } + + private static void createMissingParents(Path parent) throws IOException { + Path existing = parent; + while (existing != null && !Files.exists(existing, LinkOption.NOFOLLOW_LINKS)) { + existing = existing.getParent(); + } + if (existing == null || Files.isSymbolicLink(existing) + || !Files.isDirectory(existing, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("Setup file parent path is unsafe"); + } + Path current = existing; + for (Path segment : existing.relativize(parent)) { + current = current.resolve(segment); + createDirectoryWithoutFollowingLinks(current); + } + } + + private static void createDirectoryWithoutFollowingLinks(Path directory) throws IOException { + try { + Files.createDirectory(directory); + } catch (FileAlreadyExistsException ignored) { + // A concurrent creator is safe only when the resulting entry is a real directory. + } + if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("Setup file parent path is unsafe"); + } + } + + private static void createSafeDescendants(Path trustedRoot, Path parent) throws IOException { + Path current = trustedRoot; + for (Path segment : trustedRoot.relativize(parent)) { + current = current.resolve(segment); + createDirectoryWithoutFollowingLinks(current); + } + } + + private static void writeAndForce(FileChannel channel, byte[] content) throws IOException { + ByteBuffer buffer = ByteBuffer.wrap(content); + while (buffer.hasRemaining()) { + channel.write(buffer); + } + channel.force(true); + } + + private static Path absolute(Path path) { + return path.toAbsolutePath().normalize(); } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/unattended/SetupPasswordFileLoader.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/unattended/SetupPasswordFileLoader.java index a9279c9015..c27d35be7f 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/unattended/SetupPasswordFileLoader.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/unattended/SetupPasswordFileLoader.java @@ -18,14 +18,10 @@ package org.apache.hertzbeat.manager.setup.unattended; import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; -import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; import org.springframework.core.env.Environment; /** Loads unattended passwords exclusively from bounded owner-only files. */ @@ -34,21 +30,17 @@ public final class SetupPasswordFileLoader { public Password read(Path path) { Path normalized = path.toAbsolutePath().normalize(); + Path declaredMountRoot = normalized.getParent(); + if (declaredMountRoot == null) { + throw new IllegalStateException("Setup password file is unavailable"); + } byte[] encoded = null; char[] decoded = null; try { - if (!SecureSetupFile.isOwnerOnlyRegularFile(normalized)) { - throw new IllegalStateException("Setup password file is unavailable"); - } - long size = Files.size(normalized); - if (size <= 0 || size > MAX_PASSWORD_BYTES) { - throw new IllegalStateException("Setup password file size is invalid"); - } - encoded = Files.readAllBytes(normalized); - CharBuffer buffer = StandardCharsets.UTF_8.newDecoder().decode(ByteBuffer.wrap(encoded)); - decoded = new char[buffer.remaining()]; - buffer.get(decoded); - int length = withoutLineEnding(decoded); + encoded = SecureSetupFile.readOwnerOnly(declaredMountRoot, normalized, MAX_PASSWORD_BYTES); + decoded = new char[encoded.length]; + int decodedLength = Utf8SecretDecoder.decode(encoded, decoded); + int length = withoutLineEnding(decoded, decodedLength); if (length == 0) { throw new IllegalStateException("Setup password file is empty"); } @@ -77,8 +69,8 @@ public final class SetupPasswordFileLoader { return Path.of(file); } - private static int withoutLineEnding(char[] value) { - int length = value.length; + private static int withoutLineEnding(char[] value, int decodedLength) { + int length = decodedLength; if (length > 0 && value[length - 1] == '\n') { length--; } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/unattended/Utf8SecretDecoder.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/unattended/Utf8SecretDecoder.java new file mode 100644 index 0000000000..dd1f1755cc --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/unattended/Utf8SecretDecoder.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.unattended; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CoderResult; +import java.nio.charset.StandardCharsets; + +/** Decodes UTF-8 directly into caller-owned storage so no hidden secret character array is created. */ +final class Utf8SecretDecoder { + + private Utf8SecretDecoder() { + } + + static int decode(byte[] encoded, char[] destination) throws CharacterCodingException { + ByteBuffer input = ByteBuffer.wrap(encoded); + CharBuffer output = CharBuffer.wrap(destination); + java.nio.charset.CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder(); + CoderResult result = decoder.decode(input, output, true); + if (result.isError()) { + result.throwException(); + } + result = decoder.flush(output); + if (result.isError()) { + result.throwException(); + } + return output.position(); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/NioManagedFilePublisherTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/NioManagedFilePublisherTest.java index 333c5491df..635a22d8fa 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/NioManagedFilePublisherTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/NioManagedFilePublisherTest.java @@ -18,13 +18,20 @@ package org.apache.hertzbeat.manager.setup.config; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assumptions.assumeTrue; import java.io.IOException; +import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -84,4 +91,31 @@ class NioManagedFilePublisherTest { assertEquals("permission denied", failure.getMessage()); assertEquals(0, directoryForces.get()); } + + @Test + void ownerOnlyPublicationFailsWhenFileSystemHasNoPosixOrAclBoundary() throws Exception { + Path archive = temporaryDirectory.resolve("basic-only.zip"); + URI uri = URI.create("jar:" + archive.toUri()); + try (FileSystem fileSystem = FileSystems.newFileSystem(uri, Map.of("create", "true"))) { + Path target = fileSystem.getPath("/managed-secrets.properties"); + NioManagedFilePublisher publisher = new NioManagedFilePublisher(); + + IOException failure = assertThrows(IOException.class, () -> publisher.publish( + target, "secret-content".getBytes(StandardCharsets.UTF_8), true)); + + assertEquals("Owner-only file permissions are unavailable", failure.getMessage()); + assertFalse(Files.exists(target)); + } + } + + @Test + void ownerOnlyPublicationSetsAndVerifiesPosixPermissions() throws Exception { + assumeTrue(Files.getFileStore(temporaryDirectory).supportsFileAttributeView("posix")); + Path target = temporaryDirectory.resolve("managed-secrets.properties"); + + new NioManagedFilePublisher().publish( + target, "secret-content".getBytes(StandardCharsets.UTF_8), true); + + assertEquals(PosixFilePermissions.fromString("rw-------"), Files.getPosixFilePermissions(target)); + } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/installation/InstallationConvergenceServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/installation/InstallationConvergenceServiceTest.java index b119ce3661..097d477445 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/installation/InstallationConvergenceServiceTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/installation/InstallationConvergenceServiceTest.java @@ -36,28 +36,29 @@ class InstallationConvergenceServiceTest { void matchingDatabaseAndLocalFingerprintsAreRequiredForFullRuntime() throws Exception { Path fingerprintPath = temporaryDirectory.resolve("fingerprint"); InstallationFingerprint fingerprint = new LocalInstallationFingerprintStore( - fingerprintPath, new SecureRandom()).create(); + temporaryDirectory, fingerprintPath, new SecureRandom()).create(); InstallationRecordRepository records = mock(InstallationRecordRepository.class); when(records.findById(InstallationRecord.SINGLETON_ID)) .thenReturn(Optional.of(new InstallationRecord(fingerprint.value()))); - assertThat(new InstallationConvergenceService(records, fingerprintPath).classify()) + assertThat(new InstallationConvergenceService(records, temporaryDirectory, fingerprintPath).classify()) .isEqualTo(InstallationMode.FULL); when(records.findById(InstallationRecord.SINGLETON_ID)) .thenReturn(Optional.of(new InstallationRecord("f".repeat(64)))); - assertThat(new InstallationConvergenceService(records, fingerprintPath).classify()) + assertThat(new InstallationConvergenceService(records, temporaryDirectory, fingerprintPath).classify()) .isEqualTo(InstallationMode.RECOVERY); } @Test void fingerprintWrittenBeforeDatabaseRecordRemainsGatedAndCanConverge() throws Exception { Path fingerprintPath = temporaryDirectory.resolve("fingerprint"); - new LocalInstallationFingerprintStore(fingerprintPath, new SecureRandom()).create(); + new LocalInstallationFingerprintStore( + temporaryDirectory, fingerprintPath, new SecureRandom()).create(); InstallationRecordRepository records = mock(InstallationRecordRepository.class); when(records.findById(InstallationRecord.SINGLETON_ID)).thenReturn(Optional.empty()); - assertThat(new InstallationConvergenceService(records, fingerprintPath).classify()) + assertThat(new InstallationConvergenceService(records, temporaryDirectory, fingerprintPath).classify()) .isEqualTo(InstallationMode.UPGRADE); } @@ -67,7 +68,7 @@ class InstallationConvergenceServiceTest { Files.writeString(fingerprintPath, "not-a-fingerprint"); InstallationRecordRepository records = mock(InstallationRecordRepository.class); - assertThat(new InstallationConvergenceService(records, fingerprintPath).classify()) + assertThat(new InstallationConvergenceService(records, temporaryDirectory, fingerprintPath).classify()) .isEqualTo(InstallationMode.RECOVERY); } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/installation/InstallationPersistenceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/installation/InstallationPersistenceTest.java index 29ea4c13d7..8dfd0f6800 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/installation/InstallationPersistenceTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/installation/InstallationPersistenceTest.java @@ -93,7 +93,8 @@ class InstallationPersistenceTest { @Test void localFingerprintIsOwnerOnlyReadableAndCollisionSafe() throws Exception { Path path = temporaryDirectory.resolve("installation-id"); - LocalInstallationFingerprintStore store = new LocalInstallationFingerprintStore(path, new SecureRandom()); + LocalInstallationFingerprintStore store = new LocalInstallationFingerprintStore( + temporaryDirectory, path, new SecureRandom()); InstallationFingerprint created = store.create(); assertEquals(Optional.of(created), store.read()); assertEquals(PosixFilePermissions.fromString("rw-------"), Files.getPosixFilePermissions(path)); @@ -106,7 +107,8 @@ class InstallationPersistenceTest { Files.writeString(target, FIRST.value()); Path link = temporaryDirectory.resolve("installation-id"); Files.createSymbolicLink(link, target); - LocalInstallationFingerprintStore store = new LocalInstallationFingerprintStore(link, new SecureRandom()); + LocalInstallationFingerprintStore store = new LocalInstallationFingerprintStore( + temporaryDirectory, link, new SecureRandom()); assertEquals(Optional.empty(), store.read()); assertThrows(java.nio.file.FileAlreadyExistsException.class, store::create); } @@ -114,10 +116,25 @@ class InstallationPersistenceTest { @Test void localFingerprintRejectsPermissionsThatExposeItToOtherUsers() throws Exception { Path path = temporaryDirectory.resolve("installation-id"); - LocalInstallationFingerprintStore store = new LocalInstallationFingerprintStore(path, new SecureRandom()); + LocalInstallationFingerprintStore store = new LocalInstallationFingerprintStore( + temporaryDirectory, path, new SecureRandom()); store.create(); Files.setPosixFilePermissions(path, PosixFilePermissions.fromString("rw-r--r--")); - assertEquals(Optional.empty(), store.read()); + assertThrows(java.io.IOException.class, store::read); + } + + @Test + void localFingerprintReadRejectsAncestorSymlinkOutsideInstallationRoot() throws Exception { + Path installationRoot = Files.createDirectory(temporaryDirectory.resolve("installation")); + Path outside = Files.createDirectory(temporaryDirectory.resolve("outside")); + Path outsideFingerprint = outside.resolve("fingerprint"); + Files.writeString(outsideFingerprint, FIRST.value()); + Files.setPosixFilePermissions(outsideFingerprint, PosixFilePermissions.fromString("r--------")); + Files.createSymbolicLink(installationRoot.resolve("data"), outside); + LocalInstallationFingerprintStore store = new LocalInstallationFingerprintStore( + installationRoot, installationRoot.resolve("data/fingerprint"), new SecureRandom()); + + assertThrows(java.io.IOException.class, store::read); } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/OwnerOnlyFilePermissionsTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/OwnerOnlyFilePermissionsTest.java new file mode 100644 index 0000000000..71fa56f466 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/OwnerOnlyFilePermissionsTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.security; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.attribute.AclEntry; +import java.nio.file.attribute.AclEntryPermission; +import java.nio.file.attribute.AclEntryType; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.UserPrincipal; +import java.util.List; +import org.junit.jupiter.api.Test; + +class OwnerOnlyFilePermissionsTest { + + @Test + void aclCreationAttributeStartsWithNoAccessInsteadOfGrantingParentOwner() { + FileAttribute attribute = OwnerOnlyFilePermissions.aclCreationAttribute(); + + assertEquals("acl:acl", attribute.name()); + @SuppressWarnings("unchecked") + List entries = (List) attribute.value(); + assertTrue(entries.isEmpty()); + } + + @Test + void readableAclRequiresOwnerReadAndRejectsNonOwnerDataMutationGrants() { + UserPrincipal owner = () -> "owner"; + UserPrincipal other = () -> "other"; + AclEntry ownerRead = allow(owner, AclEntryPermission.READ_DATA); + + assertTrue(OwnerOnlyFilePermissions.isReadableOwnerOnlyAcl(List.of(ownerRead), owner)); + assertFalse(OwnerOnlyFilePermissions.isReadableOwnerOnlyAcl(List.of( + ownerRead, + allow(other, AclEntryPermission.WRITE_DATA), + allow(other, AclEntryPermission.APPEND_DATA)), owner)); + } + + @Test + void readableAclRejectsEveryNonOwnerAllowEntry() { + UserPrincipal owner = () -> "owner"; + UserPrincipal other = () -> "other"; + AclEntry ownerRead = allow(owner, AclEntryPermission.READ_DATA); + + for (AclEntryPermission permission : AclEntryPermission.values()) { + assertFalse(OwnerOnlyFilePermissions.isReadableOwnerOnlyAcl( + List.of(ownerRead, allow(other, permission)), owner), permission::name); + } + } + + private static AclEntry allow(UserPrincipal principal, AclEntryPermission... permissions) { + return AclEntry.newBuilder().setType(AclEntryType.ALLOW) + .setPrincipal(principal).setPermissions(permissions).build(); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/RemoteSetupUnlockTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/RemoteSetupUnlockTest.java index ebc0d80996..a2ed4d52c5 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/RemoteSetupUnlockTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/RemoteSetupUnlockTest.java @@ -25,8 +25,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.io.IOException; import java.net.InetAddress; -import java.nio.file.DirectoryNotEmptyException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.PosixFilePermissions; @@ -55,7 +55,8 @@ class RemoteSetupUnlockTest { void ownerOnlyCodeIsSingleUseAndBecomesStrictHttpOnlyCookie() throws Exception { Path codeFile = temporaryDirectory.resolve("unlock"); Clock clock = Clock.fixed(Instant.parse("2026-08-08T00:00:00Z"), ZoneOffset.UTC); - RemoteSetupUnlock unlock = new RemoteSetupUnlock(codeFile, clock, deterministicRandom()); + RemoteSetupUnlock unlock = new RemoteSetupUnlock( + temporaryDirectory, codeFile, clock, deterministicRandom()); unlock.open(); String code = Files.readString(codeFile); assertTrue(Files.getPosixFilePermissions(codeFile) @@ -75,6 +76,64 @@ class RemoteSetupUnlockTest { assertFalse(session.toString().contains(session.token())); } + @Test + void openRejectsTargetParentSymlinkOutsideInstallationRoot() throws Exception { + Path installationRoot = Files.createDirectory(temporaryDirectory.resolve("installation")); + Path outside = Files.createDirectory(temporaryDirectory.resolve("outside")); + Files.createDirectory(outside.resolve("config")); + Path outsideCodeFile = outside.resolve("config/unlock"); + Files.writeString(outsideCodeFile, "existing-secret"); + Files.setPosixFilePermissions(outsideCodeFile, PosixFilePermissions.fromString("rw-------")); + Files.createSymbolicLink(installationRoot.resolve("data"), outside); + Path codeFile = installationRoot.resolve("data/config/unlock"); + RemoteSetupUnlock unlock = new RemoteSetupUnlock( + installationRoot, codeFile, Clock.systemUTC(), deterministicRandom()); + + assertThrows(java.io.IOException.class, unlock::open); + assertTrue(Files.exists(outsideCodeFile)); + } + + @Test + void redeemRejectsAncestorReplacementAndDoesNotDeleteOutsideRoot() throws Exception { + Path installationRoot = Files.createDirectory(temporaryDirectory.resolve("installation")); + Path data = Files.createDirectory(installationRoot.resolve("data")); + Path codeFile = data.resolve("unlock"); + RemoteSetupUnlock unlock = new RemoteSetupUnlock( + installationRoot, codeFile, Clock.systemUTC(), deterministicRandom()); + unlock.open(); + String code = Files.readString(codeFile); + Path originalData = installationRoot.resolve("original-data"); + Files.move(data, originalData); + Path outside = Files.createDirectory(temporaryDirectory.resolve("outside")); + Path outsideCodeFile = outside.resolve("unlock"); + Files.writeString(outsideCodeFile, "outside-secret"); + Files.setPosixFilePermissions(outsideCodeFile, PosixFilePermissions.fromString("rw-------")); + Files.createSymbolicLink(data, outside); + + assertThrows(java.io.IOException.class, + () -> unlock.redeem("198.51.100.4", new SetupUnlockCode(code.toCharArray()))); + assertTrue(Files.exists(outsideCodeFile)); + } + + @Test + void closeRejectsAncestorReplacementAndDoesNotDeleteOutsideRoot() throws Exception { + Path installationRoot = Files.createDirectory(temporaryDirectory.resolve("installation")); + Path data = Files.createDirectory(installationRoot.resolve("data")); + Path codeFile = data.resolve("unlock"); + RemoteSetupUnlock unlock = new RemoteSetupUnlock( + installationRoot, codeFile, Clock.systemUTC(), deterministicRandom()); + unlock.open(); + Files.move(data, installationRoot.resolve("original-data")); + Path outside = Files.createDirectory(temporaryDirectory.resolve("outside")); + Path outsideCodeFile = outside.resolve("unlock"); + Files.writeString(outsideCodeFile, "outside-secret"); + Files.setPosixFilePermissions(outsideCodeFile, PosixFilePermissions.fromString("rw-------")); + Files.createSymbolicLink(data, outside); + + assertThrows(java.io.IOException.class, unlock::close); + assertTrue(Files.exists(outsideCodeFile)); + } + @Test void limitsRepeatedInvalidProofs() throws Exception { RemoteSetupUnlock unlock = unlock(temporaryDirectory.resolve("unlock")); @@ -94,7 +153,8 @@ class RemoteSetupUnlockTest { Clock clock = mock(Clock.class); when(clock.instant()).thenReturn(openedAt, openedAt.plus(Duration.ofMinutes(16))); Path codeFile = temporaryDirectory.resolve("unlock"); - RemoteSetupUnlock unlock = new RemoteSetupUnlock(codeFile, clock, deterministicRandom()); + RemoteSetupUnlock unlock = new RemoteSetupUnlock( + temporaryDirectory, codeFile, clock, deterministicRandom()); unlock.open(); SetupUnlockRejected expired = assertThrows(SetupUnlockRejected.class, @@ -109,11 +169,11 @@ class RemoteSetupUnlockTest { Path codeFile = temporaryDirectory.resolve("unlock"); Clock clock = Clock.fixed(Instant.parse("2026-08-08T00:00:00Z"), ZoneOffset.UTC); SecureRandom random = deterministicRandom(); - RemoteSetupUnlock firstProcess = new RemoteSetupUnlock(codeFile, clock, random); + RemoteSetupUnlock firstProcess = new RemoteSetupUnlock(temporaryDirectory, codeFile, clock, random); firstProcess.open(); String staleCode = Files.readString(codeFile); - RemoteSetupUnlock restarted = new RemoteSetupUnlock(codeFile, clock, random); + RemoteSetupUnlock restarted = new RemoteSetupUnlock(temporaryDirectory, codeFile, clock, random); restarted.open(); String replacementCode = Files.readString(codeFile); @@ -129,7 +189,8 @@ class RemoteSetupUnlockTest { void openingNewProofInvalidatesPreviousSession() throws Exception { Path codeFile = temporaryDirectory.resolve("unlock"); Clock clock = Clock.fixed(Instant.parse("2026-08-08T00:00:00Z"), ZoneOffset.UTC); - RemoteSetupUnlock unlock = new RemoteSetupUnlock(codeFile, clock, deterministicRandom()); + RemoteSetupUnlock unlock = new RemoteSetupUnlock( + temporaryDirectory, codeFile, clock, deterministicRandom()); unlock.open(); SetupAccessSession oldSession = unlock.redeem( "198.51.100.4", new SetupUnlockCode(Files.readString(codeFile).toCharArray())); @@ -143,7 +204,8 @@ class RemoteSetupUnlockTest { void ensureOpenPreservesAnActiveSession() throws Exception { Path codeFile = temporaryDirectory.resolve("unlock"); MutableClock clock = new MutableClock(Instant.parse("2026-08-08T00:00:00Z")); - RemoteSetupUnlock unlock = new RemoteSetupUnlock(codeFile, clock, deterministicRandom()); + RemoteSetupUnlock unlock = new RemoteSetupUnlock( + temporaryDirectory, codeFile, clock, deterministicRandom()); unlock.ensureOpen(); SetupAccessSession session = unlock.redeem( "198.51.100.4", new SetupUnlockCode(Files.readString(codeFile).toCharArray())); @@ -158,7 +220,8 @@ class RemoteSetupUnlockTest { void ensureOpenRenewsProofAfterExpiry() throws Exception { Path codeFile = temporaryDirectory.resolve("unlock"); MutableClock clock = new MutableClock(Instant.parse("2026-08-08T00:00:00Z")); - RemoteSetupUnlock unlock = new RemoteSetupUnlock(codeFile, clock, deterministicRandom()); + RemoteSetupUnlock unlock = new RemoteSetupUnlock( + temporaryDirectory, codeFile, clock, deterministicRandom()); unlock.ensureOpen(); String expiredCode = Files.readString(codeFile); SetupAccessSession expiredSession = unlock.redeem( @@ -182,7 +245,7 @@ class RemoteSetupUnlockTest { Files.createDirectory(codeFile); Files.writeString(codeFile.resolve("blocker"), "keep-directory-non-empty"); - assertThrows(DirectoryNotEmptyException.class, + assertThrows(IOException.class, () -> unlock.redeem("198.51.100.4", new SetupUnlockCode(code.toCharArray()))); Files.delete(codeFile.resolve("blocker")); @@ -193,7 +256,7 @@ class RemoteSetupUnlockTest { } private static RemoteSetupUnlock unlock(Path path) { - return new RemoteSetupUnlock(path, + return new RemoteSetupUnlock(path.getParent(), path, Clock.fixed(Instant.parse("2026-08-08T00:00:00Z"), ZoneOffset.UTC), deterministicRandom()); } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SecureFileChannelReaderTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SecureFileChannelReaderTest.java new file mode 100644 index 0000000000..d051ee3ac0 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SecureFileChannelReaderTest.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.security; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import org.junit.jupiter.api.Test; + +class SecureFileChannelReaderTest { + + @Test + void wipesDestinationWhenChannelReadFails() { + byte[] destination = new byte[12]; + ReadableByteChannel channel = new ReadableByteChannel() { + private boolean firstRead = true; + + @Override + public int read(ByteBuffer target) throws IOException { + if (!firstRead) { + throw new IOException("injected read failure"); + } + firstRead = false; + target.put("secret".getBytes(java.nio.charset.StandardCharsets.UTF_8)); + return 6; + } + + @Override + public boolean isOpen() { + return true; + } + + @Override + public void close() { + } + }; + + assertThrows(IOException.class, + () -> SecureFileChannelReader.readAndValidate(channel, destination, () -> { })); + + assertArrayEquals(new byte[destination.length], destination); + } + + @Test + void wipesDestinationOnShortEndOfFile() { + byte[] destination = new byte[12]; + ReadableByteChannel channel = Channels.newChannel(new ByteArrayInputStream(new byte[] {1, 2, 3})); + + assertThrows(IOException.class, + () -> SecureFileChannelReader.readAndValidate(channel, destination, () -> { })); + + assertArrayEquals(new byte[destination.length], destination); + } + + @Test + void failsAndWipesInsteadOfRetryingZeroByteRead() { + byte[] destination = new byte[6]; + ReadableByteChannel channel = new ReadableByteChannel() { + private boolean firstRead = true; + + @Override + public int read(ByteBuffer target) { + if (firstRead) { + firstRead = false; + return 0; + } + target.put("secret".getBytes(java.nio.charset.StandardCharsets.UTF_8)); + return 6; + } + + @Override + public boolean isOpen() { + return true; + } + + @Override + public void close() { + } + }; + + assertThrows(IOException.class, + () -> SecureFileChannelReader.readAndValidate(channel, destination, () -> { })); + + assertArrayEquals(new byte[destination.length], destination); + } + + @Test + void wipesDestinationWhenPostReadValidationFails() { + byte[] destination = new byte[6]; + ReadableByteChannel channel = Channels.newChannel(new ByteArrayInputStream("secret".getBytes( + java.nio.charset.StandardCharsets.UTF_8))); + + assertThrows(IOException.class, () -> SecureFileChannelReader.readAndValidate( + channel, destination, () -> { + throw new IOException("injected validation failure"); + })); + + assertArrayEquals(new byte[destination.length], destination); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileTest.java new file mode 100644 index 0000000000..def5c980cd --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileTest.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.security; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assumptions.assumeFalse; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import java.nio.charset.StandardCharsets; +import java.nio.file.FileStore; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.attribute.AclEntry; +import java.nio.file.attribute.AclEntryPermission; +import java.nio.file.attribute.AclEntryType; +import java.nio.file.attribute.AclFileAttributeView; +import java.nio.file.attribute.PosixFileAttributeView; +import java.nio.file.attribute.UserPrincipal; +import java.util.List; +import java.util.Set; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class SecureSetupFileTest { + @TempDir + private Path temporaryDirectory; + + @Test + void createsWithActualOwnerOnlyAclOnAclOnlyFileStore() throws Exception { + FileStore fileStore = Files.getFileStore(temporaryDirectory); + assumeFalse(fileStore.supportsFileAttributeView(PosixFileAttributeView.class)); + assumeTrue(fileStore.supportsFileAttributeView(AclFileAttributeView.class)); + Path target = temporaryDirectory.resolve("secret"); + byte[] content = "secret-content".getBytes(StandardCharsets.UTF_8); + + SecureSetupFile.create(temporaryDirectory, target, content); + + UserPrincipal actualOwner = Files.getOwner(target, LinkOption.NOFOLLOW_LINKS); + AclFileAttributeView aclView = Files.getFileAttributeView( + target, AclFileAttributeView.class, LinkOption.NOFOLLOW_LINKS); + assertNotNull(aclView); + AclEntry expectedOwnerEntry = AclEntry.newBuilder().setType(AclEntryType.ALLOW) + .setPrincipal(actualOwner).setPermissions(Set.of( + AclEntryPermission.READ_DATA, + AclEntryPermission.WRITE_DATA, + AclEntryPermission.APPEND_DATA, + AclEntryPermission.READ_ATTRIBUTES, + AclEntryPermission.READ_ACL, + AclEntryPermission.SYNCHRONIZE)) + .build(); + assertEquals(List.of(expectedOwnerEntry), aclView.getAcl()); + assertArrayEquals(content, Files.readAllBytes(target)); + } + + @Test + void createRejectsSymlinkInAncestorPath() throws Exception { + Path realDirectory = temporaryDirectory.resolve("real"); + Files.createDirectory(realDirectory); + Path linkedDirectory = temporaryDirectory.resolve("linked"); + Files.createSymbolicLink(linkedDirectory, realDirectory); + + assertThrows(java.io.IOException.class, () -> SecureSetupFile.create( + temporaryDirectory, + linkedDirectory.resolve("nested").resolve("secret"), + "secret-content".getBytes(StandardCharsets.UTF_8))); + } + + @Test + void createRejectsSymlinkAncestorWhenAllDescendantDirectoriesExist() throws Exception { + Path trustedRoot = Files.createDirectory(temporaryDirectory.resolve("trusted")); + Path outside = Files.createDirectory(temporaryDirectory.resolve("outside")); + Files.createDirectory(outside.resolve("nested")); + Path linkedDirectory = trustedRoot.resolve("linked"); + Files.createSymbolicLink(linkedDirectory, outside); + + assertThrows(java.io.IOException.class, () -> SecureSetupFile.create( + trustedRoot, + linkedDirectory.resolve("nested").resolve("secret"), + "secret-content".getBytes(StandardCharsets.UTF_8))); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SetupHttpUnlockServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SetupHttpUnlockServiceTest.java index 650aaf9215..584b229871 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SetupHttpUnlockServiceTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SetupHttpUnlockServiceTest.java @@ -52,7 +52,7 @@ class SetupHttpUnlockServiceTest { SetupPhase.COMPLETE, SetupAccess.LOCKED, true, "operator"); try (SetupHttpUnlockService service = new SetupHttpUnlockService( - new RemoteSetupUnlock(codeFile, Clock.systemUTC(), new SecureRandom()), + new RemoteSetupUnlock(codeFile.getParent(), codeFile, Clock.systemUTC(), new SecureRandom()), InetAddress.getByName("0.0.0.0"), state, Clock.systemUTC())) { assertThat(service.requiresUnlock()).isFalse(); assertThat(Files.exists(codeFile)).isFalse(); @@ -65,7 +65,7 @@ class SetupHttpUnlockServiceTest { SetupRuntimeState state = new SetupRuntimeState(Clock.systemUTC(), mock(ManagedConfigCapability.class), SetupPhase.CONFIGURATION_REQUIRED, SetupAccess.LOCAL, false, null); try (SetupHttpUnlockService service = new SetupHttpUnlockService( - new RemoteSetupUnlock(codeFile, Clock.systemUTC(), new SecureRandom()), + new RemoteSetupUnlock(codeFile.getParent(), codeFile, Clock.systemUTC(), new SecureRandom()), InetAddress.getLoopbackAddress(), state, Clock.systemUTC())) { MockHttpServletRequest direct = new MockHttpServletRequest("GET", "/api/setup/status"); assertThat(service.requiresUnlock(direct)).isFalse(); @@ -79,7 +79,7 @@ class SetupHttpUnlockServiceTest { SetupRuntimeState state = new SetupRuntimeState(Clock.systemUTC(), mock(ManagedConfigCapability.class), SetupPhase.CONFIGURATION_REQUIRED, SetupAccess.LOCAL, false, null); try (SetupHttpUnlockService service = new SetupHttpUnlockService( - new RemoteSetupUnlock(codeFile, Clock.systemUTC(), new SecureRandom()), + new RemoteSetupUnlock(codeFile.getParent(), codeFile, Clock.systemUTC(), new SecureRandom()), InetAddress.getLoopbackAddress(), state, Clock.systemUTC())) { MockHttpServletRequest direct = new MockHttpServletRequest("GET", "/api/setup/status"); assertThat(service.requiresUnlock(direct)).isFalse(); @@ -105,7 +105,7 @@ class SetupHttpUnlockServiceTest { SetupRuntimeState state = new SetupRuntimeState(Clock.systemUTC(), mock(ManagedConfigCapability.class), SetupPhase.CONFIGURATION_REQUIRED, SetupAccess.LOCAL, false, null); try (SetupHttpUnlockService service = new SetupHttpUnlockService( - new RemoteSetupUnlock(codeFile, Clock.systemUTC(), new SecureRandom()), + new RemoteSetupUnlock(codeFile.getParent(), codeFile, Clock.systemUTC(), new SecureRandom()), InetAddress.getLoopbackAddress(), state, Clock.systemUTC())) { MockHttpServletRequest forwarded = new MockHttpServletRequest( "POST", "/api/setup/unlock"); @@ -146,7 +146,7 @@ class SetupHttpUnlockServiceTest { SetupRuntimeState state = new SetupRuntimeState(clock, mock(ManagedConfigCapability.class), SetupPhase.CONFIGURATION_REQUIRED, SetupAccess.LOCKED, false, null); try (SetupHttpUnlockService service = new SetupHttpUnlockService( - new RemoteSetupUnlock(codeFile, clock, new SecureRandom()), + new RemoteSetupUnlock(codeFile.getParent(), codeFile, clock, new SecureRandom()), InetAddress.getByName("0.0.0.0"), state, clock)) { MockHttpServletRequest request = new MockHttpServletRequest("POST", "/api/setup/unlock"); request.setRemoteAddr("198.51.100.4"); @@ -170,7 +170,7 @@ class SetupHttpUnlockServiceTest { SetupRuntimeState state = new SetupRuntimeState(clock, mock(ManagedConfigCapability.class), SetupPhase.CONFIGURATION_REQUIRED, SetupAccess.LOCKED, false, null); try (SetupHttpUnlockService service = new SetupHttpUnlockService( - new RemoteSetupUnlock(codeFile, clock, new SecureRandom()), + new RemoteSetupUnlock(codeFile.getParent(), codeFile, clock, new SecureRandom()), InetAddress.getByName("0.0.0.0"), state, clock)) { MockHttpServletRequest request = new MockHttpServletRequest("POST", "/api/setup/unlock"); request.setRemoteAddr("198.51.100.7"); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/SetupPasswordFileLoaderTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/SetupPasswordFileLoaderTest.java index 2a9c817c9b..2d632dde65 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/SetupPasswordFileLoaderTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/SetupPasswordFileLoaderTest.java @@ -59,18 +59,49 @@ class SetupPasswordFileLoaderTest { } @Test - void rejectsNonOwnerFileAndSymlink() throws Exception { - Path file = temporaryDirectory.resolve("password"); + void rejectsNonOwnerFileAndOutOfRootSymlink() throws Exception { + Path mount = Files.createDirectory(temporaryDirectory.resolve("mount")); + Path file = mount.resolve("password"); Files.writeString(file, "secret"); Files.setPosixFilePermissions(file, PosixFilePermissions.fromString("rw-r--r--")); SetupPasswordFileLoader loader = new SetupPasswordFileLoader(); assertThrows(IllegalStateException.class, () -> loader.read(file)); - Path target = temporaryDirectory.resolve("target"); + Path target = temporaryDirectory.resolve("outside-target"); Files.writeString(target, "secret"); Files.setPosixFilePermissions(target, PosixFilePermissions.fromString("rw-------")); - Path link = temporaryDirectory.resolve("link"); + Path link = mount.resolve("password-link"); Files.createSymbolicLink(link, target); assertThrows(IllegalStateException.class, () -> loader.read(link)); } + + @Test + void readsKubernetesStyleInRootSymlinkAndFollowsRotation() throws Exception { + Path mount = Files.createDirectory(temporaryDirectory.resolve("mount")); + Path firstVersion = Files.createDirectory(mount.resolve("..2026_08_09_01")); + Path secondVersion = Files.createDirectory(mount.resolve("..2026_08_09_02")); + writeOwnerReadOnly(firstVersion.resolve("password"), "first-secret"); + writeOwnerReadOnly(secondVersion.resolve("password"), "second-secret"); + Path dataLink = mount.resolve("..data"); + Files.createSymbolicLink(dataLink, firstVersion.getFileName()); + Path passwordLink = mount.resolve("password"); + Files.createSymbolicLink(passwordLink, Path.of("..data", "password")); + + SetupPasswordFileLoader loader = new SetupPasswordFileLoader(); + try (SetupPasswordFileLoader.Password password = loader.read(passwordLink)) { + assertArrayEquals("first-secret".toCharArray(), password.copy()); + } + + Files.delete(dataLink); + Files.createSymbolicLink(dataLink, secondVersion.getFileName()); + + try (SetupPasswordFileLoader.Password password = loader.read(passwordLink)) { + assertArrayEquals("second-secret".toCharArray(), password.copy()); + } + } + + private static void writeOwnerReadOnly(Path target, String value) throws Exception { + Files.writeString(target, value); + Files.setPosixFilePermissions(target, PosixFilePermissions.fromString("r--------")); + } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/Utf8SecretDecoderTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/Utf8SecretDecoderTest.java new file mode 100644 index 0000000000..c39cc412d7 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/unattended/Utf8SecretDecoderTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.unattended; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import org.junit.jupiter.api.Test; + +class Utf8SecretDecoderTest { + + @Test + void decodesDirectlyIntoCallerOwnedWipeableArray() throws Exception { + byte[] encoded = "sëcret".getBytes(StandardCharsets.UTF_8); + char[] destination = new char[encoded.length]; + + int length = Utf8SecretDecoder.decode(encoded, destination); + + assertEquals(6, length); + assertArrayEquals("sëcret".toCharArray(), Arrays.copyOf(destination, length)); + Arrays.fill(destination, '\0'); + assertArrayEquals(new char[destination.length], destination); + } +} From ea09f43580fc6c7cccbdf7a9acdded72a9b3e870 Mon Sep 17 00:00:00 2001 From: Logic Date: Sun, 9 Aug 2026 13:00:52 +0800 Subject: [PATCH 28/71] Align startup setup contracts --- .../startup/runtime/LocalInstallationStartupProbe.java | 2 +- .../startup/config/ManagedConfigDataPrecedenceTest.java | 3 ++- .../startup/runtime/LocalInstallationStartupProbeTest.java | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbe.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbe.java index bc561bc9d0..3ec7bb77ff 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbe.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbe.java @@ -73,7 +73,7 @@ public final class LocalInstallationStartupProbe implements StartupDecisionProbe private static FingerprintState fingerprintState(Path root) { Path path = root.resolve("data/config/.installation-fingerprint"); try { - boolean present = new LocalInstallationFingerprintStore(path, new SecureRandom()).read().isPresent(); + boolean present = new LocalInstallationFingerprintStore(root, path, new SecureRandom()).read().isPresent(); if (present) { return FingerprintState.PRESENT; } diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/config/ManagedConfigDataPrecedenceTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/config/ManagedConfigDataPrecedenceTest.java index 495edfd42d..8edfb3b5f7 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/config/ManagedConfigDataPrecedenceTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/config/ManagedConfigDataPrecedenceTest.java @@ -224,7 +224,8 @@ class ManagedConfigDataPrecedenceTest { private static ManagedApplicationConfig managedApplicationWithOptions() { ManagedOptionalConfiguration options = new ManagedOptionalConfiguration( - Optional.of(new ManagedOptionalConfiguration.ServerInstrumentationSettings( + Optional.of(new ManagedOptionalConfiguration.PublicAccessSettings( + Optional.empty(), Optional.of("http://server.example.test:4318"), Optional.of("https://server.example.test:4317"))), Optional.of(new ManagedOptionalConfiguration.RetentionSettings(30)), diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbeTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbeTest.java index 5195e0ff65..2d3349ea62 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbeTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbeTest.java @@ -55,8 +55,8 @@ class LocalInstallationStartupProbeTest { new ManagedConfigurationTransaction(root).apply(bundle()); assertEquals(RuntimeMode.FULL_SETUP_GATED, new LocalInstallationStartupProbe(root, false).probe(new String[0]).mode()); - new LocalInstallationFingerprintStore(root.resolve("data/config/.installation-fingerprint"), - new SecureRandom()).create(); + new LocalInstallationFingerprintStore(root, + root.resolve("data/config/.installation-fingerprint"), new SecureRandom()).create(); assertEquals(RuntimeMode.FULL_SETUP_GATED, new LocalInstallationStartupProbe(root, false).probe(new String[0]).mode()); } From 50b17faa55bdd8e499b9cf94bf26f049f421ac39 Mon Sep 17 00:00:00 2001 From: Logic Date: Sun, 9 Aug 2026 15:00:17 +0800 Subject: [PATCH 29/71] Make setup runtime transitions durable --- .../setup/api/SetupApiConfiguration.java | 20 +- .../FileSetupTransitionIntentLock.java | 108 ++++++ .../FileSetupTransitionIntentStore.java | 182 ++++++++++ .../runtime/SetupResponseTransition.java | 2 +- .../SetupResponseTransitionFilter.java | 19 +- .../setup/runtime/SetupRuntimeTransition.java | 4 +- .../SetupRuntimeTransitionScheduler.java | 252 ++++++++++++-- .../runtime/SetupTransitionIntentStore.java | 41 +++ .../setup/security/SecureSetupFile.java | 21 +- .../workflow/SetupTransitionService.java | 23 +- .../FileSetupTransitionIntentStoreTest.java | 169 +++++++++ .../SetupResponseTransitionFilterTest.java | 103 +++++- .../SetupRuntimeTransitionDurabilityTest.java | 329 ++++++++++++++++++ .../SetupRuntimeTransitionSchedulerTest.java | 17 +- .../workflow/DefaultSetupWorkflowTest.java | 31 +- .../HeadlessSetupCoordinatorTest.java | 4 +- .../workflow/SetupTransitionServiceTest.java | 93 ++++- .../runtime/HertzBeatStartupCoordinator.java | 18 +- .../runtime/SetupTransitionSplitLockTest.java | 125 +++++++ .../HertzBeatStartupCoordinatorTest.java | 22 ++ 20 files changed, 1506 insertions(+), 77 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/FileSetupTransitionIntentLock.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/FileSetupTransitionIntentStore.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupTransitionIntentStore.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/FileSetupTransitionIntentStoreTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransitionDurabilityTest.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupTransitionSplitLockTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java index 2f385540d0..64de8851cf 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiConfiguration.java @@ -23,8 +23,8 @@ import java.nio.file.Path; import java.security.SecureRandom; import java.time.Clock; import java.util.Optional; -import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import org.apache.hertzbeat.common.runtime.BusinessRuntimeGate; import org.apache.hertzbeat.common.runtime.RuntimeMode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; @@ -37,9 +37,11 @@ import org.apache.hertzbeat.manager.setup.identity.IdentityInitializationService import org.apache.hertzbeat.manager.setup.installation.InstallationCompletionService; import org.apache.hertzbeat.manager.setup.installation.InstallationRecordRepository; import org.apache.hertzbeat.manager.setup.installation.LocalInstallationFingerprintStore; +import org.apache.hertzbeat.manager.setup.runtime.FileSetupTransitionIntentStore; import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; import org.apache.hertzbeat.manager.setup.runtime.SetupResponseTransition; import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransitionScheduler; +import org.apache.hertzbeat.manager.setup.runtime.SetupTransitionIntentStore; import org.apache.hertzbeat.manager.setup.security.RemoteSetupUnlock; import org.apache.hertzbeat.manager.setup.security.SetupHttpUnlockService; import org.apache.hertzbeat.manager.setup.unattended.SetupPasswordFileLoader; @@ -84,10 +86,15 @@ public class SetupApiConfiguration { @Bean(destroyMethod = "close") public SetupRuntimeTransitionScheduler setupRuntimeTransitionScheduler( - SetupRuntimeTransition transition) { - ExecutorService executor = Executors.newSingleThreadExecutor( + SetupRuntimeTransition transition, SetupTransitionIntentStore intents) { + ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor( Thread.ofPlatform().name("setup-runtime-transition").factory()); - return new SetupRuntimeTransitionScheduler(transition, executor); + return new SetupRuntimeTransitionScheduler(transition, intents, executor); + } + + @Bean + public SetupTransitionIntentStore setupTransitionIntentStore(Environment environment) { + return new FileSetupTransitionIntentStore(SetupInstallationPaths.root(environment)); } @Bean @@ -147,10 +154,11 @@ public class SetupApiConfiguration { SetupConfigurationCoordinator configuration, ManagedConfigCapability capability, SetupOptionsCoordinator options, ObjectProvider identityProvider, - ObjectProvider installationProvider, Environment environment) { + ObjectProvider installationProvider, + SetupTransitionIntentStore transitionIntents, Environment environment) { return new SetupTransitionService(state, validator, configuration, capability, options, identityProvider.stream().findFirst(), - completion(environment, installationProvider.stream().findFirst())); + completion(environment, installationProvider.stream().findFirst()), transitionIntents); } @Bean diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/FileSetupTransitionIntentLock.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/FileSetupTransitionIntentLock.java new file mode 100644 index 0000000000..a9a413d2b1 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/FileSetupTransitionIntentLock.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.runtime; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.ReentrantLock; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; + +/** Cooperative owner-only serialization for transition intent marker operations. */ +final class FileSetupTransitionIntentLock { + private static final String RELATIVE_PATH = "data/config/.setup-transition-intent.lock"; + private static final ConcurrentMap JVM_LOCKS = new ConcurrentHashMap<>(); + private final Path installationRoot; + private final Path lockFile; + private final ReentrantLock jvmLock; + private boolean initialized; + + FileSetupTransitionIntentLock(Path installationRoot) { + this(installationRoot, RELATIVE_PATH); + } + + FileSetupTransitionIntentLock(Path installationRoot, String relativePath) { + this.installationRoot = installationRoot.toAbsolutePath().normalize(); + lockFile = this.installationRoot.resolve(relativePath).normalize(); + if (!lockFile.startsWith(this.installationRoot)) { + throw new IllegalArgumentException("Setup transition intent lock must remain inside the installation root"); + } + // FileChannel.lock throws OverlappingFileLockException instead of waiting inside one JVM. + jvmLock = JVM_LOCKS.computeIfAbsent(lockFile, ignored -> new ReentrantLock(true)); + } + + T execute(IoOperation operation) throws IOException { + jvmLock.lock(); + try { + initialize(); + validate(); + try (FileChannel channel = FileChannel.open( + lockFile, Set.of(StandardOpenOption.WRITE, LinkOption.NOFOLLOW_LINKS)); + FileLock ignored = channel.lock()) { + return operation.run(); + } + } finally { + jvmLock.unlock(); + } + } + + void execute(IoAction action) throws IOException { + execute(() -> { + action.run(); + return null; + }); + } + + private void initialize() throws IOException { + if (initialized) { + return; + } + try { + SecureSetupFile.create(installationRoot, lockFile, new byte[] {'l', 'o', 'c', 'k', '\n'}); + } catch (FileAlreadyExistsException existing) { + // The monotonic marker protocol remains correct if deployment replaces this lock inode. + } + validate(); + SecureSetupFile.forceParentDirectoryIfSupported(installationRoot, lockFile); + initialized = true; + } + + private void validate() throws IOException { + if (!SecureSetupFile.existsInsideRootWithoutLinks(installationRoot, lockFile) + || !SecureSetupFile.isOwnerOnlyRegularFile(lockFile)) { + throw new IOException("Setup transition intent lock is invalid"); + } + } + + @FunctionalInterface + interface IoOperation { + T run() throws IOException; + } + + @FunctionalInterface + interface IoAction { + void run() throws IOException; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/FileSetupTransitionIntentStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/FileSetupTransitionIntentStore.java new file mode 100644 index 0000000000..dc59787e58 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/FileSetupTransitionIntentStore.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.runtime; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Objects; +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; + +/** Owner-only, monotonic managed-file adapter for pending setup runtime transitions. */ +public final class FileSetupTransitionIntentStore implements SetupTransitionIntentStore { + static final String RELATIVE_PATH = "data/config/setup-transition-configuration"; + static final String COMPLETION_RELATIVE_PATH = + "data/config/setup-transition-completion"; + static final String TERMINAL_RELATIVE_PATH = + "data/config/setup-transition-closed"; + private static final String INSTALLATION_CLOSED = "INSTALLATION_CLOSED"; + private static final int MAXIMUM_BYTES = 64; + private final Path installationRoot; + private final Marker configurationMarker; + private final Marker completionMarker; + private final Marker terminalMarker; + private final ParentDirectorySync parentDirectorySync; + private final FileSetupTransitionIntentLock intentLock; + private final MarkerObservation markerObservation; + + public FileSetupTransitionIntentStore(Path installationRoot) { + this(installationRoot, target -> SecureSetupFile.forceParentDirectoryIfSupported( + installationRoot, target)); + } + + FileSetupTransitionIntentStore(Path installationRoot, ParentDirectorySync parentDirectorySync) { + this(installationRoot, parentDirectorySync, + new FileSetupTransitionIntentLock(installationRoot), MarkerObservation.NONE); + } + + FileSetupTransitionIntentStore( + Path installationRoot, ParentDirectorySync parentDirectorySync, + FileSetupTransitionIntentLock intentLock, MarkerObservation markerObservation) { + this.installationRoot = Objects.requireNonNull(installationRoot, "installationRoot") + .toAbsolutePath().normalize(); + configurationMarker = marker(RELATIVE_PATH, Intent.CONFIGURATION_APPLIED.name()); + completionMarker = marker(COMPLETION_RELATIVE_PATH, Intent.INSTALLATION_COMPLETED.name()); + terminalMarker = marker(TERMINAL_RELATIVE_PATH, INSTALLATION_CLOSED); + this.parentDirectorySync = Objects.requireNonNull(parentDirectorySync, "parentDirectorySync"); + this.intentLock = Objects.requireNonNull(intentLock, "intentLock"); + this.markerObservation = Objects.requireNonNull(markerObservation, "markerObservation"); + } + + @Override + public Optional load() throws IOException { + return intentLock.execute(this::loadLocked); + } + + private Optional loadLocked() throws IOException { + if (exists(terminalMarker)) { + return Optional.empty(); + } + if (exists(completionMarker)) { + return Optional.of(Intent.INSTALLATION_COMPLETED); + } + return exists(configurationMarker) + ? Optional.of(Intent.CONFIGURATION_APPLIED) : Optional.empty(); + } + + @Override + public void save(Intent requested) throws IOException { + Objects.requireNonNull(requested, "requested"); + intentLock.execute(() -> saveLocked(requested)); + } + + private void saveLocked(Intent requested) throws IOException { + if (exists(terminalMarker)) { + parentDirectorySync.force(terminalMarker.path()); + return; + } + if (requested == Intent.INSTALLATION_COMPLETED) { + create(completionMarker); + return; + } + if (exists(completionMarker)) { + parentDirectorySync.force(completionMarker.path()); + return; + } + create(configurationMarker); + } + + @Override + public void clear(Intent completed) throws IOException { + Objects.requireNonNull(completed, "completed"); + intentLock.execute(() -> clearLocked(completed)); + } + + private void clearLocked(Intent completed) throws IOException { + if (completed == Intent.INSTALLATION_COMPLETED) { + if (exists(terminalMarker)) { + parentDirectorySync.force(terminalMarker.path()); + return; + } + if (exists(completionMarker)) { + create(terminalMarker); + } + return; + } + if (exists(configurationMarker)) { + SecureSetupFile.deleteOwnerOnlyInsideRoot(installationRoot, configurationMarker.path()); + parentDirectorySync.force(configurationMarker.path()); + } + } + + private void create(Marker marker) throws IOException { + byte[] encoded = marker.value().getBytes(StandardCharsets.UTF_8); + try { + SecureSetupFile.create(installationRoot, marker.path(), encoded); + } catch (FileAlreadyExistsException concurrent) { + if (!exists(marker)) { + throw new IOException("Setup transition marker disappeared during creation"); + } + } finally { + Arrays.fill(encoded, (byte) 0); + } + parentDirectorySync.force(marker.path()); + } + + private boolean exists(Marker marker) throws IOException { + if (!Files.exists(marker.path(), LinkOption.NOFOLLOW_LINKS)) { + markerObservation.observed(marker.path(), false); + return false; + } + byte[] encoded = SecureSetupFile.readOwnerOnlyWithoutLinks( + installationRoot, marker.path(), MAXIMUM_BYTES); + try { + String value = new String(encoded, StandardCharsets.UTF_8).strip(); + if (!marker.value().equals(value)) { + throw new IOException("Setup transition marker is invalid"); + } + markerObservation.observed(marker.path(), true); + return true; + } finally { + Arrays.fill(encoded, (byte) 0); + } + } + + private Marker marker(String relativePath, String value) { + return new Marker(installationRoot.resolve(relativePath), value); + } + + private record Marker(Path path, String value) { } + + @FunctionalInterface + interface ParentDirectorySync { + void force(Path target) throws IOException; + } + + @FunctionalInterface + interface MarkerObservation { + MarkerObservation NONE = (path, present) -> { }; + + void observed(Path path, boolean present) throws IOException; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupResponseTransition.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupResponseTransition.java index 296414a097..0d12de1f84 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupResponseTransition.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupResponseTransition.java @@ -19,7 +19,7 @@ package org.apache.hertzbeat.manager.setup.runtime; import jakarta.servlet.ServletRequest; -/** Marks one successful setup response for a transition after serialization and commit. */ +/** Marks a request whose already-durable transition intent should be woken after response processing. */ public final class SetupResponseTransition { private static final String ATTRIBUTE = SetupResponseTransition.class.getName() + ".transition"; diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupResponseTransitionFilter.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupResponseTransitionFilter.java index 55fcd707b8..abc8a1a5de 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupResponseTransitionFilter.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupResponseTransitionFilter.java @@ -24,7 +24,7 @@ import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import org.springframework.web.filter.OncePerRequestFilter; -/** Commits a successful configuration response before scheduling the destructive context transition. */ +/** Flushes successful responses while always waking an already-durable runtime transition intent. */ public final class SetupResponseTransitionFilter extends OncePerRequestFilter { private final SetupRuntimeTransitionScheduler scheduler; private final SetupResponseTransition responseTransition; @@ -38,13 +38,20 @@ public final class SetupResponseTransitionFilter extends OncePerRequestFilter { @Override protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain chain) throws ServletException, IOException { - chain.doFilter(request, response); - SetupResponseTransition.Transition transition = responseTransition.consume(request); - if (transition != null) { - response.flushBuffer(); + SetupResponseTransition.Transition transition = null; + try { + chain.doFilter(request, response); + transition = responseTransition.consume(request); + if (transition != null) { + response.flushBuffer(); + } + } finally { + if (transition == null) { + transition = responseTransition.consume(request); + } if (transition == SetupResponseTransition.Transition.INSTALLATION_COMPLETED) { scheduler.installationCompleted(); - } else { + } else if (transition == SetupResponseTransition.Transition.CONFIGURATION_APPLIED) { scheduler.configurationApplied(); } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransition.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransition.java index 59c0b37458..63bca0f3e5 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransition.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransition.java @@ -20,8 +20,8 @@ package org.apache.hertzbeat.manager.setup.runtime; /** * Application boundary used by setup completion to activate the normal runtime. * - *

The transition closes the currently active Spring context. Callers must therefore invoke it from the - * asynchronous setup operation after the HTTP response has been committed, rather than from the request thread. + *

The transition closes the currently active Spring context. Callers must therefore invoke it asynchronously + * after the setup mutation has durably recorded its intent, rather than from the request thread. */ @FunctionalInterface public interface SetupRuntimeTransition { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransitionScheduler.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransitionScheduler.java index 0df3073c4f..16f5d73eef 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransitionScheduler.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransitionScheduler.java @@ -17,36 +17,137 @@ package org.apache.hertzbeat.manager.setup.runtime; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; +import java.io.IOException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import org.apache.hertzbeat.manager.setup.runtime.SetupTransitionIntentStore.Intent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.event.EventListener; -/** Serializes deferred context transitions and never runs them before application readiness. */ +/** Serializes durable setup transitions, including bounded retry and restart recovery. */ public final class SetupRuntimeTransitionScheduler implements AutoCloseable { + private static final Logger LOGGER = LoggerFactory.getLogger(SetupRuntimeTransitionScheduler.class); + private static final int MAX_TRANSITION_ATTEMPTS = 4; + private static final int MAX_CLEAR_ATTEMPTS = 4; + private static final int MAX_RECOVERY_ATTEMPTS = 4; + private static final int MAX_DISPATCH_ATTEMPTS = 4; + private static final long INITIAL_RETRY_MILLIS = 250; private final SetupRuntimeTransition transition; - private final Executor executor; + private final SetupTransitionIntentStore intents; + private final TaskDispatcher dispatcher; + private final ScheduledExecutorService executor; private boolean ready; + private boolean recoveryFinished; private boolean closed; - private Transition running; - private Transition pending; + private Intent running; + private Intent pending; + private Intent completedUncleared; + private int attempts; + private int clearAttempts; + private int recoveryAttempts; - public SetupRuntimeTransitionScheduler(SetupRuntimeTransition transition, Executor executor) { + public SetupRuntimeTransitionScheduler( + SetupRuntimeTransition transition, SetupTransitionIntentStore intents, + ScheduledExecutorService executor) { + this(transition, intents, + (task, delayMillis) -> executor.schedule(task, delayMillis, TimeUnit.MILLISECONDS), executor); + } + + SetupRuntimeTransitionScheduler( + SetupRuntimeTransition transition, SetupTransitionIntentStore intents, TaskDispatcher dispatcher) { + this(transition, intents, dispatcher, null); + } + + private SetupRuntimeTransitionScheduler( + SetupRuntimeTransition transition, SetupTransitionIntentStore intents, + TaskDispatcher dispatcher, ScheduledExecutorService executor) { this.transition = transition; + this.intents = intents; + this.dispatcher = dispatcher; this.executor = executor; } public synchronized void configurationApplied() { - request(Transition.CONFIGURATION); + request(Intent.CONFIGURATION_APPLIED); } public synchronized void installationCompleted() { - request(Transition.COMPLETION); + request(Intent.INSTALLATION_COMPLETED); } @EventListener public synchronized void onApplicationReady(ApplicationReadyEvent ignored) { + if (closed || recoveryFinished || recoveryAttempts > 0) { + return; + } + recoverIntent(); + } + + private synchronized void recoverIntent() { + if (closed || recoveryFinished) { + return; + } + recoveryAttempts++; + Intent recovered; + try { + recovered = intents.load().orElse(null); + } catch (IOException | RuntimeException failure) { + retryRecoveryLoad(); + return; + } + finishRecovery(recovered); + } + + private void retryRecoveryLoad() { + LOGGER.warn("Cannot recover the pending setup runtime transition"); + if (recoveryAttempts < MAX_RECOVERY_ATTEMPTS) { + long delayMillis = INITIAL_RETRY_MILLIS << (recoveryAttempts - 1); + if (submit(this::recoverIntent, delayMillis)) { + return; + } + LOGGER.warn("Cannot dispatch setup runtime transition recovery"); + } + finishRecovery(null); + } + + private void finishRecovery(Intent recovered) { + recoveryFinished = true; + recoveryAttempts = 0; ready = true; + if (recovered != null) { + request(recovered); + } else { + dispatchIfReady(); + } + } + + private void request(Intent requested) { + if (closed || running == Intent.INSTALLATION_COMPLETED || running == requested) { + return; + } + if (pending == Intent.INSTALLATION_COMPLETED) { + dispatchIfReady(); + return; + } + if (completedUncleared == Intent.INSTALLATION_COMPLETED) { + if (requested == Intent.INSTALLATION_COMPLETED) { + running = requested; + clearAttempts = 0; + dispatchClear(requested, 0); + } + return; + } + if (requested == completedUncleared) { + running = requested; + clearAttempts = 0; + dispatchClear(requested, 0); + return; + } + if (pending == null || requested.supersedes(pending)) { + pending = requested; + } dispatchIfReady(); } @@ -54,35 +155,121 @@ public final class SetupRuntimeTransitionScheduler implements AutoCloseable { if (closed || !ready || running != null || pending == null) { return; } - Transition selected = pending; + Intent selected = pending; pending = null; running = selected; - executor.execute(() -> run(selected)); + attempts = 0; + dispatchTransition(selected, 0); } - private void request(Transition requested) { - if (closed - || running == Transition.COMPLETION - || running == requested - || pending == Transition.COMPLETION) { + private void dispatchTransition(Intent selected, long delayMillis) { + if (!submit(() -> run(selected), delayMillis)) { + LOGGER.warn("Cannot dispatch the pending setup runtime transition"); + running = null; + attempts = 0; + retainPending(selected); + } + } + + private boolean submit(Runnable task, long delayMillis) { + for (int dispatchAttempt = 0; dispatchAttempt < MAX_DISPATCH_ATTEMPTS; dispatchAttempt++) { + try { + dispatcher.dispatch(task, delayMillis); + return true; + } catch (RuntimeException failure) { + // A rejected task was not accepted; retry is bounded and does not create another thread. + } + } + return false; + } + + private void retainPending(Intent selected) { + if (pending == null || selected.supersedes(pending)) { + pending = selected; + } + } + + private void run(Intent selected) { + synchronized (this) { + if (closed || running != selected) { + if (running == selected) { + running = null; + attempts = 0; + } + return; + } + attempts++; + } + try { + execute(selected); + } catch (RuntimeException failure) { + retry(selected); return; } - pending = requested; - dispatchIfReady(); + synchronized (this) { + completedUncleared = selected; + attempts = 0; + clearAttempts = 0; + } + clearAfterSuccess(selected); } - private void run(Transition selected) { + private void execute(Intent selected) { + if (selected == Intent.INSTALLATION_COMPLETED) { + transition.completeSetup(); + } else { + transition.configurationApplied(); + } + } + + private void retry(Intent selected) { + synchronized (this) { + LOGGER.warn("Setup runtime transition failed; its durable intent remains pending"); + if (!closed && attempts < MAX_TRANSITION_ATTEMPTS) { + dispatchTransition(selected, INITIAL_RETRY_MILLIS << (attempts - 1)); + return; + } + running = null; + attempts = 0; + dispatchIfReady(); + } + } + + private void clearAfterSuccess(Intent selected) { try { - if (selected == Transition.COMPLETION) { - transition.completeSetup(); - } else { - transition.configurationApplied(); - } - } finally { - synchronized (this) { - running = null; - dispatchIfReady(); + intents.clear(selected); + } catch (IOException | RuntimeException failure) { + retryClear(selected); + return; + } + synchronized (this) { + completedUncleared = null; + running = null; + clearAttempts = 0; + dispatchIfReady(); + } + } + + private void retryClear(Intent selected) { + synchronized (this) { + LOGGER.warn("Cannot clear the completed setup runtime transition intent"); + clearAttempts++; + if (!closed && clearAttempts < MAX_CLEAR_ATTEMPTS) { + dispatchClear(selected, INITIAL_RETRY_MILLIS << (clearAttempts - 1)); + return; } + running = null; + clearAttempts = 0; + dispatchIfReady(); + } + } + + private void dispatchClear(Intent selected, long delayMillis) { + if (!submit(() -> clearAfterSuccess(selected), delayMillis)) { + LOGGER.warn("Cannot dispatch the completed setup runtime transition clear"); + running = null; + clearAttempts = 0; + dispatchIfReady(); } } @@ -90,10 +277,13 @@ public final class SetupRuntimeTransitionScheduler implements AutoCloseable { public synchronized void close() { closed = true; pending = null; - if (executor instanceof ExecutorService service) { - service.shutdown(); + if (executor != null) { + executor.shutdown(); } } - private enum Transition { CONFIGURATION, COMPLETION } + @FunctionalInterface + interface TaskDispatcher { + void dispatch(Runnable task, long delayMillis); + } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupTransitionIntentStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupTransitionIntentStore.java new file mode 100644 index 0000000000..ed19376301 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/SetupTransitionIntentStore.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.runtime; + +import java.io.IOException; +import java.util.Optional; + +/** Durable, secret-free intent that bridges a committed setup mutation and its runtime transition. */ +public interface SetupTransitionIntentStore { + + Optional load() throws IOException; + + void save(Intent intent) throws IOException; + + void clear(Intent completed) throws IOException; + + /** Completion subsumes the earlier configuration transition and must never be downgraded. */ + enum Intent { + CONFIGURATION_APPLIED, + INSTALLATION_COMPLETED; + + boolean supersedes(Intent current) { + return this == INSTALLATION_COMPLETED && current == CONFIGURATION_APPLIED; + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFile.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFile.java index 2ca7a5d379..728e88f3b2 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFile.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFile.java @@ -26,6 +26,7 @@ import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.FileAttribute; +import java.util.Arrays; import java.util.Set; /** Secure creation, permission enforcement, and bounded reading of local setup files. */ @@ -107,6 +108,24 @@ public final class SecureSetupFile { return true; } + /** + * Forces the target's parent directory on POSIX providers after a directory-entry mutation. + * + * @return {@code false} when the provider has no POSIX directory-fsync contract; the owner-only forced-file + * guarantee remains intact on that provider + */ + public static boolean forceParentDirectoryIfSupported(Path trustedRoot, Path target) throws IOException { + Path parent = resolveWithoutLinksInsideRoot(trustedRoot, target).getParent(); + if (!Files.getFileStore(parent).supportsFileAttributeView("posix")) { + return false; + } + try (FileChannel channel = FileChannel.open( + parent, Set.of(StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS))) { + channel.force(true); + } + return true; + } + private static byte[] readResolvedOwnerOnly(Path resolvedTarget, int maximumBytes) throws IOException { if (!isOwnerOnlyRegularFile(resolvedTarget)) { throw new IOException("Setup file is not an owner-only regular file"); @@ -131,7 +150,7 @@ public final class SecureSetupFile { return transferred; } finally { if (content != null) { - java.util.Arrays.fill(content, (byte) 0); + Arrays.fill(content, (byte) 0); } } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionService.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionService.java index 337281358c..5a6c2c7aef 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionService.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionService.java @@ -7,6 +7,7 @@ package org.apache.hertzbeat.manager.setup.workflow; +import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -20,6 +21,7 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidateRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationSection; import org.apache.hertzbeat.manager.setup.api.SetupApiException; import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; @@ -28,6 +30,8 @@ import org.apache.hertzbeat.manager.setup.identity.AdministratorCredentials; import org.apache.hertzbeat.manager.setup.identity.BootstrapIdentityConflict; import org.apache.hertzbeat.manager.setup.identity.IdentityInitializationService; import org.apache.hertzbeat.manager.setup.identity.InvalidAdministratorUsername; +import org.apache.hertzbeat.manager.setup.runtime.SetupTransitionIntentStore; +import org.apache.hertzbeat.manager.setup.runtime.SetupTransitionIntentStore.Intent; import org.springframework.http.HttpStatus; /** Single state transition boundary shared by browser and headless setup adapters. */ @@ -39,12 +43,14 @@ public final class SetupTransitionService { private final SetupOptionsCoordinator options; private final Optional identities; private final Optional completion; + private final SetupTransitionIntentStore transitionIntents; public SetupTransitionService(SetupRuntimeState state, SetupRequestValidator validator, SetupConfigurationCoordinator configuration, ManagedConfigCapability capability, SetupOptionsCoordinator options, Optional identities, - Optional completion) { + Optional completion, + SetupTransitionIntentStore transitionIntents) { this.state = state; this.validator = validator; this.configuration = configuration; @@ -52,6 +58,7 @@ public final class SetupTransitionService { this.options = options; this.identities = identities; this.completion = completion; + this.transitionIntents = transitionIntents; } public ConfigurationResponse configure(ConfigurationCommand command) { @@ -63,6 +70,9 @@ public final class SetupTransitionService { state.ensurePhase(expected); command.validate(validator); ConfigurationResponse response = command.configure(configuration, capability); + if (response.phase() == SetupPhase.APPLICATION_STARTING) { + recordTransition(Intent.CONFIGURATION_APPLIED); + } state.configurationApplied(expected, response.operationId(), response.phase()); return response; } @@ -120,6 +130,7 @@ public final class SetupTransitionService { throw new SetupWorkflowConflict(); } completion.orElseThrow(SetupWorkflowConflict::new).completeInstallation(); + recordTransition(Intent.INSTALLATION_COMPLETED); state.complete(); return username; } @@ -130,6 +141,14 @@ public final class SetupTransitionService { } } + private void recordTransition(Intent intent) { + try { + transitionIntents.save(intent); + } catch (IOException | RuntimeException failure) { + throw new SetupApiException(SetupErrorCode.CONFIG_WRITE_FAILED, HttpStatus.INTERNAL_SERVER_ERROR); + } + } + /** Transport adapter for required configuration while the transition stays transport-neutral. */ public interface ConfigurationCommand { SetupPhase expectedPhase(); @@ -217,7 +236,7 @@ public final class SetupTransitionService { } private static void requireValid(SetupRequestValidator validator, ValidateRequest request) { - var response = validator.validate(request); + ValidationResponse response = validator.validate(request); if (!response.valid()) { throw new SetupApiException(response.errorCode(), HttpStatus.BAD_REQUEST); } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/FileSetupTransitionIntentStoreTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/FileSetupTransitionIntentStoreTest.java new file mode 100644 index 0000000000..3693f36944 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/FileSetupTransitionIntentStoreTest.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.runtime; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.apache.hertzbeat.manager.setup.runtime.SetupTransitionIntentStore.Intent; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class FileSetupTransitionIntentStoreTest { + @TempDir + private Path installationRoot; + + @Test + void persistsOwnerOnlyIntentAndClearsOnlyTheMatchingCompletion() throws Exception { + FileSetupTransitionIntentStore store = new FileSetupTransitionIntentStore(installationRoot); + + store.save(Intent.CONFIGURATION_APPLIED); + store.save(Intent.CONFIGURATION_APPLIED); + + assertThat(new FileSetupTransitionIntentStore(installationRoot).load()) + .contains(Intent.CONFIGURATION_APPLIED); + Path checkpoint = installationRoot.resolve(FileSetupTransitionIntentStore.RELATIVE_PATH); + assertThat(SecureSetupFile.isOwnerOnlyRegularFile(checkpoint)).isTrue(); + + store.clear(Intent.INSTALLATION_COMPLETED); + assertThat(store.load()).contains(Intent.CONFIGURATION_APPLIED); + + store.clear(Intent.CONFIGURATION_APPLIED); + assertThat(store.load()).isEmpty(); + + store.save(Intent.CONFIGURATION_APPLIED); + assertThat(store.load()).contains(Intent.CONFIGURATION_APPLIED); + } + + @Test + void completionSupersedesConfigurationAndCannotBeDowngraded() throws Exception { + FileSetupTransitionIntentStore store = new FileSetupTransitionIntentStore(installationRoot); + + store.save(Intent.CONFIGURATION_APPLIED); + store.save(Intent.INSTALLATION_COMPLETED); + store.save(Intent.CONFIGURATION_APPLIED); + + assertThat(store.load()).contains(Intent.INSTALLATION_COMPLETED); + } + + @Test + void staleConfigurationClearCannotDeleteCompletionAfterLockFileReplacement() throws Exception { + FileSetupTransitionIntentStore oldContext = new FileSetupTransitionIntentStore(installationRoot); + FileSetupTransitionIntentStore newContext = new FileSetupTransitionIntentStore(installationRoot); + Path staleConfigurationMarker = installationRoot.resolve( + FileSetupTransitionIntentStore.RELATIVE_PATH); + + oldContext.save(Intent.CONFIGURATION_APPLIED); + newContext.save(Intent.INSTALLATION_COMPLETED); + + // Models an old process finishing a stale clear after a replacement lock inode was acquired. + SecureSetupFile.deleteOwnerOnlyInsideRoot(installationRoot, staleConfigurationMarker); + + assertThat(newContext.load()).contains(Intent.INSTALLATION_COMPLETED); + } + + @Test + void staleConfigurationSaveCannotReopenCompletedInstallation() throws Exception { + FileSetupTransitionIntentStore completingContext = + new FileSetupTransitionIntentStore(installationRoot); + FileSetupTransitionIntentStore staleContext = + new FileSetupTransitionIntentStore(installationRoot); + + completingContext.save(Intent.INSTALLATION_COMPLETED); + completingContext.clear(Intent.INSTALLATION_COMPLETED); + + // Models a stale save finishing after the completion clear through a replacement lock inode. + staleContext.save(Intent.CONFIGURATION_APPLIED); + + assertThat(completingContext.load()).isEmpty(); + } + + @Test + void synchronizesTheParentDirectoryAfterEachMonotonicMarker() throws Exception { + List synchronizedEntries = new ArrayList<>(); + FileSetupTransitionIntentStore store = new FileSetupTransitionIntentStore( + installationRoot, synchronizedEntries::add); + Path configuration = installationRoot.resolve(FileSetupTransitionIntentStore.RELATIVE_PATH); + Path completion = installationRoot.resolve( + FileSetupTransitionIntentStore.COMPLETION_RELATIVE_PATH); + Path terminal = installationRoot.resolve(FileSetupTransitionIntentStore.TERMINAL_RELATIVE_PATH); + + store.save(Intent.CONFIGURATION_APPLIED); + store.save(Intent.INSTALLATION_COMPLETED); + store.clear(Intent.INSTALLATION_COMPLETED); + + assertThat(synchronizedEntries).containsExactly(configuration, completion, terminal); + assertThat(SecureSetupFile.isOwnerOnlyRegularFile(completion)).isTrue(); + assertThat(SecureSetupFile.isOwnerOnlyRegularFile(terminal)).isTrue(); + } + + @Test + void cooperatingStoreInstancesAreSerializedWithinProcess() throws Exception { + FileSetupTransitionIntentStore initial = new FileSetupTransitionIntentStore(installationRoot); + initial.save(Intent.CONFIGURATION_APPLIED); + CountDownLatch clearReachedDirectorySync = new CountDownLatch(1); + CountDownLatch allowClearToFinish = new CountDownLatch(1); + CountDownLatch upgradeStarted = new CountDownLatch(1); + FileSetupTransitionIntentStore oldContext = new FileSetupTransitionIntentStore( + installationRoot, ignored -> { + clearReachedDirectorySync.countDown(); + try { + allowClearToFinish.await(5, TimeUnit.SECONDS); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while coordinating clear", interrupted); + } + }); + FileSetupTransitionIntentStore newContext = new FileSetupTransitionIntentStore(installationRoot); + + try (ExecutorService executor = Executors.newFixedThreadPool(2)) { + Future clear = executor.submit(() -> { + oldContext.clear(Intent.CONFIGURATION_APPLIED); + return null; + }); + assertThat(clearReachedDirectorySync.await(5, TimeUnit.SECONDS)).isTrue(); + Future upgrade = executor.submit(() -> { + upgradeStarted.countDown(); + newContext.save(Intent.INSTALLATION_COMPLETED); + return null; + }); + assertThat(upgradeStarted.await(5, TimeUnit.SECONDS)).isTrue(); + + try { + assertThrows(TimeoutException.class, () -> upgrade.get(200, TimeUnit.MILLISECONDS)); + } finally { + allowClearToFinish.countDown(); + } + clear.get(5, TimeUnit.SECONDS); + upgrade.get(5, TimeUnit.SECONDS); + } + + assertThat(newContext.load()).contains(Intent.INSTALLATION_COMPLETED); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupResponseTransitionFilterTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupResponseTransitionFilterTest.java index 0eb704d186..335d9b2618 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupResponseTransitionFilterTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupResponseTransitionFilterTest.java @@ -18,24 +18,48 @@ package org.apache.hertzbeat.manager.setup.runtime; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.nio.file.Path; +import java.time.Clock; import java.util.ArrayList; import java.util.List; +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupAccess; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; +import org.apache.hertzbeat.manager.setup.runtime.SetupTransitionIntentStore.Intent; +import org.apache.hertzbeat.manager.setup.workflow.SetupConfigurationCoordinator; +import org.apache.hertzbeat.manager.setup.workflow.SetupOptionsCoordinator; +import org.apache.hertzbeat.manager.setup.workflow.SetupRequestValidator; +import org.apache.hertzbeat.manager.setup.workflow.SetupRuntimeState; +import org.apache.hertzbeat.manager.setup.workflow.SetupTransitionService; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.mock.web.MockHttpServletRequest; import org.springframework.mock.web.MockHttpServletResponse; class SetupResponseTransitionFilterTest { + @TempDir + private Path installationRoot; @Test void configurationTransitionRunsOnlyAfterResponseCommitAndApplicationReadiness() throws Exception { SetupRuntimeTransition transition = mock(SetupRuntimeTransition.class); List tasks = new ArrayList<>(); - SetupRuntimeTransitionScheduler scheduler = new SetupRuntimeTransitionScheduler(transition, tasks::add); + SetupRuntimeTransitionScheduler scheduler = new SetupRuntimeTransitionScheduler( + transition, mock(SetupTransitionIntentStore.class), (task, ignored) -> tasks.add(task)); SetupResponseTransition marker = new SetupResponseTransition(); SetupResponseTransitionFilter filter = new SetupResponseTransitionFilter(scheduler, marker); MockHttpServletRequest request = new MockHttpServletRequest("POST", "/api/setup/configuration"); @@ -55,4 +79,81 @@ class SetupResponseTransitionFilterTest { tasks.removeFirst().run(); verify(transition).configurationApplied(); } + + @Test + void committedIntentSurvivesResponseFlushFailureAndStillWakesTransition() throws Exception { + FileSetupTransitionIntentStore intents = new FileSetupTransitionIntentStore(installationRoot); + SetupRuntimeTransition transition = mock(SetupRuntimeTransition.class); + List tasks = new ArrayList<>(); + SetupRuntimeTransitionScheduler scheduler = new SetupRuntimeTransitionScheduler( + transition, intents, (task, ignored) -> tasks.add(task)); + ManagedConfigCapability capability = mock(ManagedConfigCapability.class); + SetupRuntimeState state = new SetupRuntimeState(Clock.systemUTC(), capability, + SetupPhase.CONFIGURATION_REQUIRED, SetupAccess.LOCAL, false, null); + SetupConfigurationCoordinator configuration = mock(SetupConfigurationCoordinator.class); + SetupTransitionService.ConfigurationCommand command = + mock(SetupTransitionService.ConfigurationCommand.class); + when(command.expectedPhase()).thenReturn(SetupPhase.CONFIGURATION_REQUIRED); + when(command.configure(configuration, capability)).thenReturn(new ConfigurationResponse( + "operation", SetupOperationState.AWAITING_RESTART, + SetupPhase.APPLICATION_STARTING, 1_000, false)); + SetupTransitionService transitions = new SetupTransitionService( + state, mock(SetupRequestValidator.class), configuration, capability, + mock(SetupOptionsCoordinator.class), Optional.empty(), Optional.empty(), intents); + SetupResponseTransition marker = new SetupResponseTransition(); + SetupResponseTransitionFilter filter = new SetupResponseTransitionFilter(scheduler, marker); + MockHttpServletRequest request = new MockHttpServletRequest("POST", "/api/setup/configuration"); + HttpServletResponse response = mock(HttpServletResponse.class); + doThrow(new IOException("client disconnected")).when(response).flushBuffer(); + + assertThrows(IOException.class, () -> filter.doFilter(request, response, (servletRequest, ignored) -> { + transitions.configure(command); + marker.arm(servletRequest); + })); + + assertThat(state.phase()).isEqualTo(SetupPhase.APPLICATION_STARTING); + assertThat(intents.load()).contains(Intent.CONFIGURATION_APPLIED); + scheduler.onApplicationReady(mock(ApplicationReadyEvent.class)); + assertThat(tasks).hasSize(1); + } + + @Test + void armedIntentWakesExactlyOnceWhenResponseSerializationFails() throws Exception { + List failures = List.of( + new IOException("response write failed"), + new ServletException("response serialization failed")); + for (int index = 0; index < failures.size(); index++) { + Exception responseFailure = failures.get(index); + Path root = installationRoot.resolve("failure-" + index); + FileSetupTransitionIntentStore intents = new FileSetupTransitionIntentStore(root); + intents.save(Intent.CONFIGURATION_APPLIED); + SetupRuntimeTransition transition = mock(SetupRuntimeTransition.class); + List tasks = new ArrayList<>(); + SetupRuntimeTransitionScheduler scheduler = new SetupRuntimeTransitionScheduler( + transition, intents, (task, ignored) -> tasks.add(task)); + SetupResponseTransition marker = new SetupResponseTransition(); + SetupResponseTransitionFilter filter = new SetupResponseTransitionFilter(scheduler, marker); + MockHttpServletRequest request = new MockHttpServletRequest("POST", "/api/setup/configuration"); + + Throwable propagated = assertThrows(responseFailure.getClass(), + () -> filter.doFilter(request, new MockHttpServletResponse(), (servletRequest, ignored) -> { + marker.arm(servletRequest); + if (responseFailure instanceof IOException writeFailure) { + throw writeFailure; + } + throw (ServletException) responseFailure; + })); + + assertThat(propagated).isSameAs(responseFailure); + assertThat(intents.load()).contains(Intent.CONFIGURATION_APPLIED); + assertThat(tasks).isEmpty(); + + scheduler.onApplicationReady(mock(ApplicationReadyEvent.class)); + assertThat(tasks).hasSize(1); + tasks.removeFirst().run(); + + verify(transition).configurationApplied(); + assertThat(tasks).isEmpty(); + } + } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransitionDurabilityTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransitionDurabilityTest.java new file mode 100644 index 0000000000..4cad9d7a6e --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransitionDurabilityTest.java @@ -0,0 +1,329 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.runtime; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hertzbeat.manager.setup.runtime.SetupTransitionIntentStore.Intent; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.boot.context.event.ApplicationReadyEvent; + +class SetupRuntimeTransitionDurabilityTest { + @TempDir + private Path installationRoot; + + @Test + void failedTransitionRetainsIntentAndRetriesBeforeClearingOnSuccess() throws Exception { + MemoryIntentStore intents = new MemoryIntentStore(Intent.CONFIGURATION_APPLIED); + ManualDispatcher dispatcher = new ManualDispatcher(); + AtomicInteger calls = new AtomicInteger(); + SetupRuntimeTransition transition = () -> { + if (calls.getAndIncrement() == 0) { + throw new IllegalStateException("controlled failure"); + } + }; + SetupRuntimeTransitionScheduler scheduler = new SetupRuntimeTransitionScheduler( + transition, intents, dispatcher::dispatch); + + scheduler.onApplicationReady(mock(ApplicationReadyEvent.class)); + dispatcher.runNext(); + + assertThat(intents.load()).contains(Intent.CONFIGURATION_APPLIED); + assertThat(intents.clears).isZero(); + assertThat(dispatcher.delays()).containsExactly(0L, 250L); + + dispatcher.runNext(); + + assertThat(calls).hasValue(2); + assertThat(intents.load()).isEmpty(); + assertThat(intents.clears).isOne(); + } + + @Test + void retryBackoffAndAttemptsAreBoundedWhileIntentRemainsDurable() throws Exception { + MemoryIntentStore intents = new MemoryIntentStore(Intent.INSTALLATION_COMPLETED); + ManualDispatcher dispatcher = new ManualDispatcher(); + SetupRuntimeTransition transition = () -> { + throw new IllegalStateException("controlled failure"); + }; + SetupRuntimeTransitionScheduler scheduler = new SetupRuntimeTransitionScheduler( + transition, intents, dispatcher::dispatch); + + scheduler.onApplicationReady(mock(ApplicationReadyEvent.class)); + dispatcher.runAll(); + + assertThat(dispatcher.delays()).containsExactly(0L, 250L, 500L, 1_000L); + assertThat(intents.load()).contains(Intent.INSTALLATION_COMPLETED); + assertThat(intents.clears).isZero(); + } + + @Test + void newSchedulerRecoversPendingIntentAndDuplicateSignalsDoNotDoubleExecute() throws Exception { + FileSetupTransitionIntentStore firstProcess = new FileSetupTransitionIntentStore(installationRoot); + firstProcess.save(Intent.CONFIGURATION_APPLIED); + ManualDispatcher dispatcher = new ManualDispatcher(); + SetupRuntimeTransition transition = mock(SetupRuntimeTransition.class); + SetupRuntimeTransitionScheduler restarted = new SetupRuntimeTransitionScheduler( + transition, new FileSetupTransitionIntentStore(installationRoot), dispatcher::dispatch); + + restarted.configurationApplied(); + restarted.configurationApplied(); + restarted.onApplicationReady(mock(ApplicationReadyEvent.class)); + restarted.configurationApplied(); + + assertThat(dispatcher.tasks).hasSize(1); + dispatcher.runNext(); + + verify(transition, times(1)).configurationApplied(); + assertThat(firstProcess.load()).isEmpty(); + assertThat(dispatcher.tasks).isEmpty(); + } + + @Test + void clearFailureRetriesWithoutRepeatingTransitionAndDuplicateWakeRetriesOnlyTheClear() throws Exception { + MemoryIntentStore intents = new MemoryIntentStore(Intent.INSTALLATION_COMPLETED); + intents.clearFailures = 4; + ManualDispatcher dispatcher = new ManualDispatcher(); + SetupRuntimeTransition transition = mock(SetupRuntimeTransition.class); + SetupRuntimeTransitionScheduler scheduler = new SetupRuntimeTransitionScheduler( + transition, intents, dispatcher::dispatch); + + scheduler.onApplicationReady(mock(ApplicationReadyEvent.class)); + dispatcher.runAll(); + + verify(transition, times(1)).completeSetup(); + assertThat(dispatcher.delays()).containsExactly(0L, 250L, 500L, 1_000L); + assertThat(intents.load()).contains(Intent.INSTALLATION_COMPLETED); + + intents.clearFailures = 0; + scheduler.installationCompleted(); + dispatcher.runAll(); + + verify(transition, times(1)).completeSetup(); + assertThat(intents.load()).isEmpty(); + } + + @Test + void dispatcherRejectionIsBoundedAndTransientRejectionRecoversInProcess() throws Exception { + MemoryIntentStore intents = new MemoryIntentStore(Intent.CONFIGURATION_APPLIED); + RejectingDispatcher dispatcher = new RejectingDispatcher(1); + SetupRuntimeTransition transition = mock(SetupRuntimeTransition.class); + SetupRuntimeTransitionScheduler scheduler = new SetupRuntimeTransitionScheduler( + transition, intents, dispatcher::dispatch); + + scheduler.onApplicationReady(mock(ApplicationReadyEvent.class)); + + assertThat(dispatcher.dispatchCalls).isEqualTo(2); + assertThat(dispatcher.tasks).hasSize(1); + dispatcher.tasks.removeFirst().run(); + verify(transition, times(1)).configurationApplied(); + assertThat(intents.load()).isEmpty(); + + MemoryIntentStore retained = new MemoryIntentStore(Intent.CONFIGURATION_APPLIED); + RejectingDispatcher unavailable = new RejectingDispatcher(4); + SetupRuntimeTransition recoveredTransition = mock(SetupRuntimeTransition.class); + SetupRuntimeTransitionScheduler bounded = new SetupRuntimeTransitionScheduler( + recoveredTransition, retained, unavailable::dispatch); + bounded.onApplicationReady(mock(ApplicationReadyEvent.class)); + + assertThat(unavailable.dispatchCalls).isEqualTo(4); + assertThat(retained.load()).contains(Intent.CONFIGURATION_APPLIED); + + bounded.configurationApplied(); + assertThat(unavailable.dispatchCalls).isEqualTo(5); + assertThat(unavailable.tasks).hasSize(1); + unavailable.tasks.removeFirst().run(); + + verify(recoveredTransition, times(1)).configurationApplied(); + assertThat(retained.load()).isEmpty(); + } + + @Test + void recoveryLoadRetriesCheckedAndRuntimeFailuresBeforeDispatchingTheRecoveredIntent() throws Exception { + for (Throwable firstFailure : List.of( + new IOException("controlled read failure"), + new IllegalStateException("controlled provider failure"))) { + MemoryIntentStore intents = new MemoryIntentStore(Intent.CONFIGURATION_APPLIED); + intents.loadFailure = firstFailure; + intents.loadFailures = 1; + ManualDispatcher dispatcher = new ManualDispatcher(); + SetupRuntimeTransition transition = mock(SetupRuntimeTransition.class); + SetupRuntimeTransitionScheduler scheduler = new SetupRuntimeTransitionScheduler( + transition, intents, dispatcher::dispatch); + + scheduler.onApplicationReady(mock(ApplicationReadyEvent.class)); + + assertThat(intents.loads).isOne(); + assertThat(dispatcher.delays()).containsExactly(250L); + dispatcher.runNext(); + assertThat(intents.loads).isEqualTo(2); + assertThat(dispatcher.delays()).containsExactly(250L, 0L); + dispatcher.runNext(); + + verify(transition, times(1)).configurationApplied(); + assertThat(intents.current()).isNull(); + } + } + + @Test + void recoveryLoadRetryIsBoundedAndLeavesTheDiskIntentUntouched() { + MemoryIntentStore intents = new MemoryIntentStore(Intent.INSTALLATION_COMPLETED); + intents.loadFailure = new IOException("controlled persistent read failure"); + intents.loadFailures = Integer.MAX_VALUE; + ManualDispatcher dispatcher = new ManualDispatcher(); + SetupRuntimeTransition transition = mock(SetupRuntimeTransition.class); + SetupRuntimeTransitionScheduler scheduler = new SetupRuntimeTransitionScheduler( + transition, intents, dispatcher::dispatch); + + scheduler.onApplicationReady(mock(ApplicationReadyEvent.class)); + dispatcher.runAll(); + + assertThat(intents.loads).isEqualTo(4); + assertThat(dispatcher.delays()).containsExactly(250L, 500L, 1_000L); + assertThat(intents.current()).isEqualTo(Intent.INSTALLATION_COMPLETED); + assertThat(intents.clears).isZero(); + verifyNoInteractions(transition); + } + + @Test + void recoveredConfigurationCannotStrandAnAlreadyPendingCompletion() throws Exception { + MemoryIntentStore intents = new MemoryIntentStore(Intent.CONFIGURATION_APPLIED); + intents.loadFailure = new IOException("controlled first read failure"); + intents.loadFailures = 1; + ManualDispatcher dispatcher = new ManualDispatcher(); + SetupRuntimeTransition transition = mock(SetupRuntimeTransition.class); + SetupRuntimeTransitionScheduler scheduler = new SetupRuntimeTransitionScheduler( + transition, intents, dispatcher::dispatch); + + scheduler.onApplicationReady(mock(ApplicationReadyEvent.class)); + scheduler.installationCompleted(); + dispatcher.runNext(); + + assertThat(dispatcher.delays()).containsExactly(250L, 0L); + dispatcher.runNext(); + + verify(transition, times(1)).completeSetup(); + verify(transition, times(0)).configurationApplied(); + } + + private static final class MemoryIntentStore implements SetupTransitionIntentStore { + private Intent intent; + private int clears; + private int loads; + private int loadFailures; + private Throwable loadFailure; + + private MemoryIntentStore(Intent intent) { + this.intent = intent; + } + + @Override + public Optional load() throws IOException { + loads++; + if (loadFailures > 0) { + loadFailures--; + if (loadFailure instanceof IOException checked) { + throw checked; + } + throw (RuntimeException) loadFailure; + } + return Optional.ofNullable(intent); + } + + @Override + public void save(Intent requested) { + intent = requested; + } + + @Override + public void clear(Intent completed) throws IOException { + if (clearFailures > 0) { + clearFailures--; + throw new IOException("controlled clear failure"); + } + if (intent == completed) { + intent = null; + clears++; + } + } + + private int clearFailures; + + private Intent current() { + return intent; + } + } + + private static final class ManualDispatcher { + private final List tasks = new ArrayList<>(); + private final List dispatchedDelays = new ArrayList<>(); + + private void dispatch(Runnable task, long delayMillis) { + tasks.add(new ScheduledTask(task, delayMillis)); + dispatchedDelays.add(delayMillis); + } + + private void runNext() { + tasks.removeFirst().task().run(); + } + + private void runAll() { + while (!tasks.isEmpty()) { + runNext(); + } + } + + private List delays() { + return List.copyOf(dispatchedDelays); + } + } + + private record ScheduledTask(Runnable task, long delayMillis) { + } + + private static final class RejectingDispatcher { + private final List tasks = new ArrayList<>(); + private int remainingRejections; + private int dispatchCalls; + + private RejectingDispatcher(int remainingRejections) { + this.remainingRejections = remainingRejections; + } + + private void dispatch(Runnable task, long ignored) { + dispatchCalls++; + if (remainingRejections > 0) { + remainingRejections--; + throw new RejectedExecutionException("controlled rejection"); + } + tasks.add(task); + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransitionSchedulerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransitionSchedulerTest.java index 49df5325f1..3d68745b59 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransitionSchedulerTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupRuntimeTransitionSchedulerTest.java @@ -25,8 +25,8 @@ import static org.mockito.Mockito.verify; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -39,7 +39,8 @@ class SetupRuntimeTransitionSchedulerTest { void runningConfigurationCoalescesDuplicatesAndQueuesOneHigherPriorityCompletion() { SetupRuntimeTransition transition = mock(SetupRuntimeTransition.class); List tasks = new ArrayList<>(); - SetupRuntimeTransitionScheduler scheduler = new SetupRuntimeTransitionScheduler(transition, tasks::add); + SetupRuntimeTransitionScheduler scheduler = new SetupRuntimeTransitionScheduler( + transition, mock(SetupTransitionIntentStore.class), (task, ignored) -> tasks.add(task)); scheduler.onApplicationReady(mock(ApplicationReadyEvent.class)); scheduler.configurationApplied(); @@ -60,7 +61,8 @@ class SetupRuntimeTransitionSchedulerTest { void completionSupersedesConfigurationBeforeReadiness() { SetupRuntimeTransition transition = mock(SetupRuntimeTransition.class); List tasks = new ArrayList<>(); - SetupRuntimeTransitionScheduler scheduler = new SetupRuntimeTransitionScheduler(transition, tasks::add); + SetupRuntimeTransitionScheduler scheduler = new SetupRuntimeTransitionScheduler( + transition, mock(SetupTransitionIntentStore.class), (task, ignored) -> tasks.add(task)); scheduler.configurationApplied(); scheduler.installationCompleted(); @@ -74,8 +76,8 @@ class SetupRuntimeTransitionSchedulerTest { @Test void transitionCanCloseSchedulerFromItsExecutorWithoutInterruptingOrDispatchingPendingWork() - throws InterruptedException { - ExecutorService executor = Executors.newSingleThreadExecutor(); + throws Exception { + ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); AtomicReference schedulerReference = new AtomicReference<>(); CountDownLatch transitionStarted = new CountDownLatch(1); CountDownLatch allowClose = new CountDownLatch(1); @@ -105,7 +107,9 @@ class SetupRuntimeTransitionSchedulerTest { completionCalls.incrementAndGet(); } }; - SetupRuntimeTransitionScheduler scheduler = new SetupRuntimeTransitionScheduler(transition, executor); + SetupTransitionIntentStore intents = mock(SetupTransitionIntentStore.class); + SetupRuntimeTransitionScheduler scheduler = new SetupRuntimeTransitionScheduler( + transition, intents, executor); schedulerReference.set(scheduler); scheduler.onApplicationReady(mock(ApplicationReadyEvent.class)); @@ -120,5 +124,6 @@ class SetupRuntimeTransitionSchedulerTest { assertThat(interrupted).isFalse(); assertThat(configurationCalls).hasValue(1); assertThat(completionCalls).hasValue(0); + verify(intents).clear(SetupTransitionIntentStore.Intent.CONFIGURATION_APPLIED); } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java index 270cc13bdd..12ce61d855 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java @@ -17,6 +17,7 @@ package org.apache.hertzbeat.manager.setup.workflow; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -35,7 +36,9 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.AdministratorRequest; @@ -46,6 +49,7 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationResp import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.PublicAccessConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupAccess; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; @@ -59,6 +63,7 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiException; import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; import org.apache.hertzbeat.manager.setup.config.SecretValue; import org.apache.hertzbeat.manager.setup.identity.IdentityInitializationService; +import org.apache.hertzbeat.manager.setup.runtime.SetupTransitionIntentStore; import org.junit.jupiter.api.Test; class DefaultSetupWorkflowTest { @@ -81,7 +86,7 @@ class DefaultSetupWorkflowTest { "https://server.example.test:4318", "\u0000"), null, null); - var response = workflow.configureOptions(request); + OptionsResponse response = workflow.configureOptions(request); assertTrue(response.publicBaseUrlConfigured()); assertTrue(response.serverOtlpHttpConfigured()); @@ -130,7 +135,7 @@ class DefaultSetupWorkflowTest { workflow.complete(new CompleteRequest(SetupPhase.OPTIONAL_CONFIGURATION, List.of(SetupWarningCode.H2_NON_PRODUCTION))); - org.mockito.Mockito.verify(completion).completeInstallation(); + verify(completion).completeInstallation(); } @Test @@ -161,10 +166,10 @@ class DefaultSetupWorkflowTest { OptionsRequest request = new OptionsRequest( new PublicAccessConfiguration(null, "http://collector.example.test:4318", null), null, null); - try (var executor = Executors.newFixedThreadPool(2)) { - var optionsResult = executor.submit(() -> workflow.configureOptions(request)); + try (ExecutorService executor = Executors.newFixedThreadPool(2)) { + Future optionsResult = executor.submit(() -> workflow.configureOptions(request)); persistenceStarted.await(5, TimeUnit.SECONDS); - var completionResult = executor.submit(() -> headless.complete( + Future completionResult = executor.submit(() -> headless.complete( List.of(SetupWarningCode.H2_NON_PRODUCTION))); assertThrows(TimeoutException.class, () -> completionResult.get(200, TimeUnit.MILLISECONDS)); @@ -172,7 +177,7 @@ class DefaultSetupWorkflowTest { optionsResult.get(5, TimeUnit.SECONDS); ExecutionException rejected = assertThrows(ExecutionException.class, () -> completionResult.get(5, TimeUnit.SECONDS)); - org.assertj.core.api.Assertions.assertThat(rejected.getCause()).isInstanceOf(SetupApiException.class); + assertThat(rejected.getCause()).isInstanceOf(SetupApiException.class); } verifyNoInteractions(completion); } @@ -212,10 +217,10 @@ class DefaultSetupWorkflowTest { "localhost:4001", "http://localhost:4000", "public", null, null)); try (SecretValue metadataPassword = SecretValue.of("secret"); - var executor = Executors.newFixedThreadPool(2)) { - var browserResult = executor.submit(() -> browser.configure(browserRequest)); + ExecutorService executor = Executors.newFixedThreadPool(2)) { + Future browserResult = executor.submit(() -> browser.configure(browserRequest)); configurationStarted.await(5, TimeUnit.SECONDS); - var headlessResult = executor.submit(() -> headless.configure( + Future headlessResult = executor.submit(() -> headless.configure( new HeadlessSetupWorkflow.RequiredConfiguration(SetupPhase.CONFIGURATION_REQUIRED, ApplyMode.MANAGED_WRITE, new HeadlessSetupWorkflow.Metadata(MetadataDatabaseKind.H2, @@ -228,7 +233,7 @@ class DefaultSetupWorkflowTest { browserResult.get(5, TimeUnit.SECONDS); ExecutionException rejected = assertThrows(ExecutionException.class, () -> headlessResult.get(5, TimeUnit.SECONDS)); - org.assertj.core.api.Assertions.assertThat(rejected.getCause()) + assertThat(rejected.getCause()) .isInstanceOf(SetupWorkflowConflict.class); } verify(configuration, never()).configure(any(HeadlessSetupWorkflow.RequiredConfiguration.class), @@ -242,7 +247,8 @@ class DefaultSetupWorkflowTest { Optional completion, SetupOptionsCoordinator options, Clock clock, SetupMutationSerializer mutations) { SetupTransitionService transitions = new SetupTransitionService( - state, validator, configuration, capability, options, identities, completion); + state, validator, configuration, capability, options, identities, completion, + mock(SetupTransitionIntentStore.class)); return new DefaultSetupWorkflow(state, validator, operations, clock, mutations, transitions); } @@ -254,7 +260,8 @@ class DefaultSetupWorkflowTest { Optional completion, SetupMutationSerializer mutations) { SetupTransitionService transitions = new SetupTransitionService( state, validator, configuration, capability, - mock(SetupOptionsCoordinator.class), identities, completion); + mock(SetupOptionsCoordinator.class), identities, completion, + mock(SetupTransitionIntentStore.class)); return new HeadlessSetupCoordinator(state, mutations, transitions); } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinatorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinatorTest.java index e0f7b9c6df..8953b69e00 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinatorTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HeadlessSetupCoordinatorTest.java @@ -32,6 +32,7 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; import org.apache.hertzbeat.manager.setup.api.SetupApiException; import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; import org.apache.hertzbeat.manager.setup.identity.IdentityInitializationService; +import org.apache.hertzbeat.manager.setup.runtime.SetupTransitionIntentStore; import org.junit.jupiter.api.Test; class HeadlessSetupCoordinatorTest { @@ -47,7 +48,8 @@ class HeadlessSetupCoordinatorTest { SetupTransitionService transitions = new SetupTransitionService(state, mock(SetupRequestValidator.class), mock(SetupConfigurationCoordinator.class), capability, mock(SetupOptionsCoordinator.class), - Optional.of(mock(IdentityInitializationService.class)), Optional.of(completion)); + Optional.of(mock(IdentityInitializationService.class)), Optional.of(completion), + mock(SetupTransitionIntentStore.class)); HeadlessSetupCoordinator coordinator = new HeadlessSetupCoordinator( state, new SetupMutationSerializer(), transitions); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionServiceTest.java index b83cb6d6b6..1b6311141b 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionServiceTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/SetupTransitionServiceTest.java @@ -12,12 +12,14 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.never; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; +import java.io.IOException; import java.time.Clock; import java.time.Instant; import java.time.ZoneOffset; @@ -36,11 +38,13 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseC import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionalConfigurationSummary; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionsResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.PublicAccessConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupAccess; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.StatusResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupWarningCode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreKind; @@ -53,7 +57,11 @@ import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; import org.apache.hertzbeat.manager.setup.config.SecretValue; import org.apache.hertzbeat.manager.setup.identity.IdentityInitializationService; import org.apache.hertzbeat.manager.setup.identity.BootstrapIdentityConflict; +import org.apache.hertzbeat.manager.setup.runtime.SetupTransitionIntentStore; +import org.apache.hertzbeat.manager.setup.runtime.SetupTransitionIntentStore.Intent; import org.junit.jupiter.api.Test; +import org.mockito.InOrder; +import org.springframework.http.HttpStatus; class SetupTransitionServiceTest { private static final Clock CLOCK = Clock.fixed(Instant.parse("2026-08-08T00:00:00Z"), ZoneOffset.UTC); @@ -63,7 +71,7 @@ class SetupTransitionServiceTest { ManagedConfigCapability capability = mock(ManagedConfigCapability.class); SetupRuntimeState state = state(capability, SetupPhase.CONFIGURATION_REQUIRED, false, null); SetupRequestValidator validator = mock(SetupRequestValidator.class); - when(validator.validate(any(org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidateRequest.class))) + when(validator.validate(any(ValidateRequest.class))) .thenReturn(new ValidationResponse(true, CLOCK.instant(), null, List.of())); SetupConfigurationCoordinator configuration = mock(SetupConfigurationCoordinator.class); when(configuration.configure(any(ConfigurationRequest.class), any())) @@ -86,6 +94,76 @@ class SetupTransitionServiceTest { assertThat(headlessState.phase()).isEqualTo(SetupPhase.APPLICATION_STARTING); } + @Test + void durableConfigurationAndCompletionRecordIntentBeforePublishingRuntimeState() throws Exception { + ManagedConfigCapability capability = mock(ManagedConfigCapability.class); + SetupRequestValidator validator = mock(SetupRequestValidator.class); + when(validator.validate(any(ValidateRequest.class))) + .thenReturn(new ValidationResponse(true, CLOCK.instant(), null, List.of())); + SetupConfigurationCoordinator configuration = mock(SetupConfigurationCoordinator.class); + when(configuration.configure(any(ConfigurationRequest.class), any())) + .thenReturn(configurationResponse("configuration")); + SetupTransitionIntentStore configurationIntents = mock(SetupTransitionIntentStore.class); + SetupRuntimeState configurationState = state( + capability, SetupPhase.CONFIGURATION_REQUIRED, false, null); + SetupTransitionService configurationTransitions = new SetupTransitionService( + configurationState, validator, configuration, capability, mock(SetupOptionsCoordinator.class), + Optional.empty(), Optional.empty(), configurationIntents); + + configurationTransitions.configure( + SetupTransitionService.ConfigurationCommand.browser(browserConfiguration())); + + InOrder configurationOrder = inOrder(configuration, configurationIntents); + configurationOrder.verify(configuration).configure(any(ConfigurationRequest.class), any()); + configurationOrder.verify(configurationIntents).save(Intent.CONFIGURATION_APPLIED); + assertThat(configurationState.phase()).isEqualTo(SetupPhase.APPLICATION_STARTING); + + SetupCompletionCoordinator completion = mock(SetupCompletionCoordinator.class); + SetupTransitionIntentStore completionIntents = mock(SetupTransitionIntentStore.class); + SetupRuntimeState completionState = state( + capability, SetupPhase.OPTIONAL_CONFIGURATION, true, "operator"); + SetupTransitionService completionTransitions = new SetupTransitionService( + completionState, validator, configuration, capability, mock(SetupOptionsCoordinator.class), + Optional.empty(), Optional.of(completion), completionIntents); + + completionTransitions.complete(SetupTransitionService.CompletionCommand.headless( + completionState.pendingWarnings())); + + InOrder order = inOrder(completion, completionIntents); + order.verify(completion).completeInstallation(); + order.verify(completionIntents).save(Intent.INSTALLATION_COMPLETED); + assertThat(completionState.phase()).isEqualTo(SetupPhase.COMPLETE); + } + + @Test + void transitionIntentFailuresUseStableErrorWithoutPublishingRuntimeState() throws Exception { + for (Throwable storeFailure : List.of( + new IOException("private checkpoint path"), + new IllegalStateException("private provider failure"))) { + ManagedConfigCapability capability = mock(ManagedConfigCapability.class); + SetupRuntimeState state = state(capability, SetupPhase.CONFIGURATION_REQUIRED, false, null); + SetupRequestValidator validator = mock(SetupRequestValidator.class); + when(validator.validate(any(ValidateRequest.class))) + .thenReturn(new ValidationResponse(true, CLOCK.instant(), null, List.of())); + SetupConfigurationCoordinator configuration = mock(SetupConfigurationCoordinator.class); + when(configuration.configure(any(ConfigurationRequest.class), any())) + .thenReturn(configurationResponse("configuration")); + SetupTransitionIntentStore intents = mock(SetupTransitionIntentStore.class); + doThrow(storeFailure).when(intents).save(Intent.CONFIGURATION_APPLIED); + SetupTransitionService transitions = new SetupTransitionService( + state, validator, configuration, capability, mock(SetupOptionsCoordinator.class), + Optional.empty(), Optional.empty(), intents); + + assertThatThrownBy(() -> transitions.configure( + SetupTransitionService.ConfigurationCommand.browser(browserConfiguration()))) + .isInstanceOfSatisfying(SetupApiException.class, failure -> { + assertThat(failure.errorCode()).isEqualTo(SetupErrorCode.CONFIG_WRITE_FAILED); + assertThat(failure).hasMessage("config_write_failed").hasNoCause(); + }); + assertThat(state.phase()).isEqualTo(SetupPhase.CONFIGURATION_REQUIRED); + } + } + @Test void browserAndHeadlessAdministratorCommandsRejectTheSameWrongPhaseBeforeSideEffects() { ManagedConfigCapability capability = mock(ManagedConfigCapability.class); @@ -165,7 +243,7 @@ class SetupTransitionServiceTest { void externalApplyReentryIsExplicitAndStillRunsValidationForBothTransports() { ManagedConfigCapability capability = mock(ManagedConfigCapability.class); SetupRequestValidator validator = mock(SetupRequestValidator.class); - when(validator.validate(any(org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidateRequest.class))) + when(validator.validate(any(ValidateRequest.class))) .thenReturn(new ValidationResponse(true, CLOCK.instant(), null, List.of())); SetupConfigurationCoordinator configuration = mock(SetupConfigurationCoordinator.class); ConfigurationResponse response = new ConfigurationResponse("replacement", @@ -249,11 +327,11 @@ class SetupTransitionServiceTest { .thenReturn(new ValidationResponse(true, CLOCK.instant(), null, List.of())); SetupOptionsCoordinator options = mock(SetupOptionsCoordinator.class); doThrow(new SetupApiException(SetupErrorCode.CONFIG_WRITE_FAILED, - org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR)).when(options).persist(any()); + HttpStatus.INTERNAL_SERVER_ERROR)).when(options).persist(any()); SetupTransitionService transitions = transitions(state, validator, mock(SetupConfigurationCoordinator.class), capability, options, Optional.empty(), Optional.empty()); - var before = state.status(); + StatusResponse before = state.status(); assertThatThrownBy(() -> transitions.configureOptions(optionsRequest())) .isInstanceOf(SetupApiException.class); @@ -286,7 +364,7 @@ class SetupTransitionServiceTest { new MailConfiguration("mail.example.test", 25, MailSecurity.NONE, null, null, "alerts@example.test")); - var response = transitions.configureOptions(request); + OptionsResponse response = transitions.configureOptions(request); assertThat(response.publicBaseUrlConfigured()).isTrue(); assertThat(response.serverOtlpHttpConfigured()).isFalse(); @@ -333,7 +411,8 @@ class SetupTransitionServiceTest { SetupOptionsCoordinator options, Optional identities, Optional completion) { return new SetupTransitionService( - state, validator, configuration, capability, options, identities, completion); + state, validator, configuration, capability, options, identities, completion, + mock(SetupTransitionIntentStore.class)); } private static void assertOptionsValidationFailure( @@ -347,7 +426,7 @@ class SetupTransitionServiceTest { SetupTransitionService transitions = transitions(state, validator, mock(SetupConfigurationCoordinator.class), capability, options, Optional.empty(), Optional.empty()); - var before = state.status(); + StatusResponse before = state.status(); assertThatThrownBy(() -> transitions.configureOptions(request)) .isInstanceOfSatisfying(SetupApiException.class, diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java index 0d82c96862..7b1438f10b 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java @@ -29,6 +29,7 @@ public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition private final StartupFailureReporter failureReporter; private String[] args = new String[0]; private RunningApplicationContext currentContext; + private boolean normalRuntimeSelected; public HertzBeatStartupCoordinator(StartupDecisionProbe probe, StartupContextLauncher launcher) { this(probe, launcher, new StartupFailureReporter()); @@ -55,7 +56,13 @@ public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition @Override public synchronized void configurationApplied() { - transition(new StartupDecision(RuntimeMode.FULL_SETUP_GATED)); + if (normalRuntimeSelected) { + return; + } + // The intent may be stale; the durable startup probe remains authoritative for the target mode. + StartupDecision currentDecision = Objects.requireNonNull( + probe.probe(args.clone()), "startup decision"); + transition(currentDecision); } @Override @@ -66,6 +73,7 @@ public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition public synchronized RunningApplicationContext transition(StartupDecision decision) { Objects.requireNonNull(decision, "decision"); if (currentContext != null && currentContext.isActive() && currentContext.mode() == decision.mode()) { + recordNormalSelection(); return currentContext; } closeCurrent(); @@ -88,9 +96,17 @@ public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition throw recoveryFailure; } } + recordNormalSelection(); return currentContext; } + private void recordNormalSelection() { + if (currentContext != null && currentContext.isActive() + && currentContext.mode() == RuntimeMode.NORMAL) { + normalRuntimeSelected = true; + } + } + public synchronized RuntimeMode mode() { return currentContext == null ? null : currentContext.mode(); } diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupTransitionSplitLockTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupTransitionSplitLockTest.java new file mode 100644 index 0000000000..4f0c499f89 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupTransitionSplitLockTest.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.runtime; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.setup.runtime.SetupTransitionIntentStore.Intent; +import org.apache.hertzbeat.startup.runtime.HertzBeatStartupCoordinator; +import org.apache.hertzbeat.startup.runtime.RunningApplicationContext; +import org.apache.hertzbeat.startup.runtime.StartupContextLauncher; +import org.apache.hertzbeat.startup.runtime.StartupDecision; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class SetupTransitionSplitLockTest { + @TempDir + private Path installationRoot; + + @Test + void completionBetweenMarkerReadsCannotLetStaleContextDowngradeSharedRuntime() throws Exception { + RecordingLauncher launcher = new RecordingLauncher(); + AtomicInteger probeCalls = new AtomicInteger(); + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( + ignored -> { + probeCalls.incrementAndGet(); + return new StartupDecision(RuntimeMode.FULL_SETUP_GATED); + }, launcher); + coordinator.start(new String[0]); + new FileSetupTransitionIntentStore(installationRoot).save(Intent.CONFIGURATION_APPLIED); + + FileSetupTransitionIntentStore completingContext = storeWithLock("completion"); + AtomicBoolean interleaved = new AtomicBoolean(); + Path completionMarker = installationRoot.resolve( + FileSetupTransitionIntentStore.COMPLETION_RELATIVE_PATH); + FileSetupTransitionIntentStore staleContext = new FileSetupTransitionIntentStore( + installationRoot, ignored -> { }, + new FileSetupTransitionIntentLock( + installationRoot, "data/config/.setup-transition-stale.lock"), + (path, present) -> { + if (path.equals(completionMarker) && !present + && interleaved.compareAndSet(false, true)) { + completingContext.save(Intent.INSTALLATION_COMPLETED); + coordinator.completeSetup(); + completingContext.clear(Intent.INSTALLATION_COMPLETED); + } + }); + SetupRuntimeTransitionScheduler staleScheduler = new SetupRuntimeTransitionScheduler( + coordinator, staleContext, (task, delayMillis) -> task.run()); + + staleScheduler.onApplicationReady(null); + + assertThat(interleaved).isTrue(); + assertThat(coordinator.mode()).isEqualTo(RuntimeMode.NORMAL); + assertThat(probeCalls).hasValue(1); + assertThat(launcher.events).containsExactly( + "open:full_setup_gated", "close:full_setup_gated", "open:normal"); + } + + private FileSetupTransitionIntentStore storeWithLock(String identity) { + return new FileSetupTransitionIntentStore( + installationRoot, ignored -> { }, + new FileSetupTransitionIntentLock( + installationRoot, "data/config/.setup-transition-" + identity + ".lock"), + FileSetupTransitionIntentStore.MarkerObservation.NONE); + } + + private static final class RecordingLauncher implements StartupContextLauncher { + private final List events = new ArrayList<>(); + + @Override + public RunningApplicationContext launch( + StartupDecision decision, String[] args, SetupRuntimeTransition transition) { + events.add("open:" + decision.mode().value()); + return new RecordingContext(decision.mode(), events); + } + } + + private static final class RecordingContext implements RunningApplicationContext { + private final RuntimeMode mode; + private final List events; + private boolean active = true; + + private RecordingContext(RuntimeMode mode, List events) { + this.mode = mode; + this.events = events; + } + + @Override + public RuntimeMode mode() { + return mode; + } + + @Override + public boolean isActive() { + return active; + } + + @Override + public void close() { + active = false; + events.add("close:" + mode.value()); + } + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinatorTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinatorTest.java index 2e90753aab..bde5811bbc 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinatorTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinatorTest.java @@ -25,6 +25,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.hertzbeat.common.runtime.RuntimeMode; import org.apache.hertzbeat.manager.setup.config.SetupInstallationPaths; import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; @@ -96,6 +97,27 @@ class HertzBeatStartupCoordinatorTest { assertFalse(first.isActive()); } + @Test + void normalLaunchFallbackToRecoveryCanReprobeAndConvergeAfterConfiguration() { + RecordingLauncher launcher = new RecordingLauncher(); + launcher.failMode = RuntimeMode.NORMAL; + AtomicInteger probeCalls = new AtomicInteger(); + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( + ignored -> probeCalls.getAndIncrement() == 0 + ? StartupDecision.normal() + : new StartupDecision(RuntimeMode.FULL_SETUP_GATED), launcher); + + coordinator.start(new String[0]); + launcher.failMode = null; + coordinator.configurationApplied(); + + assertEquals(2, probeCalls.get()); + assertEquals(RuntimeMode.FULL_SETUP_GATED, coordinator.mode()); + assertEquals(List.of( + "open:normal", "open:recovery", "close:recovery", "open:full_setup_gated"), + launcher.events); + } + @Test void launchFailureClosesOldContextAndFallsBackToRecovery() { RecordingLauncher launcher = new RecordingLauncher(); From 52948160129056a0aff9e9be8ee1fa767a3bfe37 Mon Sep 17 00:00:00 2001 From: Logic Date: Sun, 9 Aug 2026 15:05:44 +0800 Subject: [PATCH 30/71] Align setup login redirect --- .../hertzbeat/manager/setup/api/SetupApiContract.java | 1 + .../manager/setup/workflow/DefaultSetupWorkflow.java | 4 +++- .../manager/setup/workflow/DefaultSetupWorkflowTest.java | 7 +++++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java index 717b1564a5..c4253504e6 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java @@ -44,6 +44,7 @@ public final class SetupApiContract { public static final String OPTIONS_PATH = "/api/setup/options"; public static final String EXPORT_PATH = "/api/setup/export"; public static final String COMPLETE_PATH = "/api/setup/complete"; + public static final String LOGIN_PATH = "/passport/login"; private SetupApiContract() { } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflow.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflow.java index 93b6cf8174..f61e854472 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflow.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflow.java @@ -17,6 +17,8 @@ package org.apache.hertzbeat.manager.setup.workflow; +import static org.apache.hertzbeat.manager.setup.api.SetupApiContract.LOGIN_PATH; + import java.time.Clock; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.AdministratorRequest; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.AdministratorResponse; @@ -129,7 +131,7 @@ public final class DefaultSetupWorkflow implements SetupWorkflow { private CompleteResponse completeMutation(CompleteRequest request) { String username = transitions.complete(SetupTransitionService.CompletionCommand.browser(request)); - return new CompleteResponse(SetupPhase.COMPLETE, clock.instant(), "/login", username); + return new CompleteResponse(SetupPhase.COMPLETE, clock.instant(), LOGIN_PATH, username); } private void requireWritable() { diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java index 12ce61d855..3860ff459a 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultSetupWorkflowTest.java @@ -44,6 +44,7 @@ import java.util.concurrent.TimeoutException; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.AdministratorRequest; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.CompleteRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.CompleteResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationRequest; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigurationResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; @@ -133,8 +134,10 @@ class DefaultSetupWorkflowTest { () -> workflow.complete(new CompleteRequest(SetupPhase.OPTIONAL_CONFIGURATION, List.of()))); verifyNoInteractions(completion); - workflow.complete(new CompleteRequest(SetupPhase.OPTIONAL_CONFIGURATION, - List.of(SetupWarningCode.H2_NON_PRODUCTION))); + CompleteResponse response = workflow.complete(new CompleteRequest( + SetupPhase.OPTIONAL_CONFIGURATION, List.of(SetupWarningCode.H2_NON_PRODUCTION))); + + assertThat(response.loginPath()).isEqualTo("/passport/login"); verify(completion).completeInstallation(); } From ad45b32881a42f4d931933780184e31cd57c07fd Mon Sep 17 00:00:00 2001 From: Logic Date: Sun, 9 Aug 2026 16:09:57 +0800 Subject: [PATCH 31/71] Define metadata migration contract --- .../src/test/resources/sureness.yml | 6 + .../setup/api/DeploymentApiContract.java | 176 +++++++++--- .../setup/api/DeploymentController.java | 118 ++++++++ .../manager/setup/api/DeploymentWorkflow.java | 8 +- .../setup/api/MigrationContractValidator.java | 192 +++++++++++++ .../manager/setup/api/SetupApiContract.java | 4 + .../setup/api/SetupExceptionHandler.java | 2 +- .../workflow/MetadataMigrationPolicy.java | 76 +++++ .../workflow/MigrationExportRenderer.java | 29 ++ ...eploymentRouteAuthorizationConfigTest.java | 83 ++++++ .../setup/api/DeploymentApiContractTest.java | 193 ++++++++++++- .../DeploymentControllerRegistrationTest.java | 65 +++++ .../setup/api/DeploymentControllerTest.java | 259 ++++++++++++++++++ .../workflow/MetadataMigrationPolicyTest.java | 155 +++++++++++ .../src/test/resources/sureness.yml | 6 + .../src/main/resources/sureness.yml | 6 + .../hertzbeat-mysql-iotdb/conf/sureness.yml | 6 + .../conf/sureness.yml | 6 + .../conf/sureness.yml | 6 + .../conf/sureness.yml | 6 + .../conf/sureness.yml | 6 + script/sureness.yml | 6 + 22 files changed, 1366 insertions(+), 48 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentController.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/MigrationContractValidator.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationPolicy.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationExportRenderer.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/DeploymentRouteAuthorizationConfigTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerRegistrationTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationPolicyTest.java diff --git a/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/sureness.yml b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/sureness.yml index d84b4c6dd7..ba078f99b1 100644 --- a/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/sureness.yml +++ b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/sureness.yml @@ -76,6 +76,12 @@ resourceRole: - /api/notice/**===post===[admin,user] - /api/notice/**===put===[admin,user] - /api/notice/**===delete===[admin] + - /api/config/deployment===get===[admin] + - /api/config/deployment/validate===post===[admin] + - /api/config/deployment/metadata-migrations===post===[admin] + - /api/config/deployment/metadata-migrations/*===get===[admin] + - /api/config/deployment/metadata-migrations/*/activate===post===[admin] + - /api/config/deployment/metadata-migrations/*/export===post===[admin] - /api/config/email===get===[admin,user,guest] - /api/config/email===post===[admin] - /api/config/sms===get===[admin,user,guest] diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContract.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContract.java index 89a451dcd3..7d69d2d357 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContract.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContract.java @@ -19,17 +19,19 @@ package org.apache.hertzbeat.manager.setup.api; import com.fasterxml.jackson.annotation.JsonValue; import jakarta.validation.Valid; +import jakarta.validation.constraints.Max; +import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotNull; import jakarta.validation.constraints.PositiveOrZero; import java.time.Instant; +import java.util.Locale; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportFormat; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ManagementDatabaseSummary; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreSummary; /** Authenticated deployment configuration and H2 migration contract. */ @@ -42,6 +44,8 @@ public final class DeploymentApiContract { "/api/config/deployment/metadata-migrations/{operationId}"; public static final String ACTIVATE_PATH = "/api/config/deployment/metadata-migrations/{operationId}/activate"; + public static final String EXPORT_PATH = + "/api/config/deployment/metadata-migrations/{operationId}/export"; private DeploymentApiContract() { } @@ -49,48 +53,95 @@ public final class DeploymentApiContract { private interface WireValue { @JsonValue - String value(); + default String value() { + return ((Enum) this).name().toLowerCase(Locale.ROOT); + } + } + + /** Whether maintenance mode currently protects migration writes. */ + public enum MaintenanceMode implements WireValue { + INACTIVE, + ACTIVE + } + + /** Deployment shape relevant to migration safety. */ + public enum DeploymentTopology implements WireValue { + SINGLE_NODE, + MULTI_NODE, + UNKNOWN + } + + /** Target schema inspection result supplied by a database-specific adapter. */ + public enum TargetInspection implements WireValue { + EMPTY, + NON_EMPTY, + UNKNOWN + } + + /** Migration-specific lifecycle; ready-to-activate is deliberately non-terminal. */ + public enum MigrationOperationState implements WireValue { + PENDING, + RUNNING, + READY_TO_ACTIVATE, + AWAITING_EXTERNAL_APPLY, + AWAITING_RESTART, + SUCCEEDED, + FAILED, + ROLLED_BACK } /** Supported external metadata migration target. */ public enum MigrationTarget implements WireValue { - MYSQL("mysql", MetadataDatabaseKind.MYSQL), - POSTGRESQL("postgresql", MetadataDatabaseKind.POSTGRESQL); + MYSQL(MetadataDatabaseKind.MYSQL), + POSTGRESQL(MetadataDatabaseKind.POSTGRESQL); - private final String value; private final MetadataDatabaseKind databaseKind; - MigrationTarget(String value, MetadataDatabaseKind databaseKind) { - this.value = value; + MigrationTarget(MetadataDatabaseKind databaseKind) { this.databaseKind = databaseKind; } - @Override - public String value() { - return value; - } - MetadataDatabaseKind databaseKind() { return databaseKind; } } + /** Operator-visible stage without table names, SQL, or verification evidence. */ + public enum MigrationStage implements WireValue { + QUEUED, + COPYING, + VERIFYING, + READY_TO_ACTIVATE, + AWAITING_EXTERNAL_APPLY, + ACTIVATING, + AWAITING_RESTART, + COMPLETED, + ROLLING_BACK, + ROLLED_BACK, + FAILED + } + /** Migration verification lifecycle. */ public enum VerificationState implements WireValue { - PENDING("pending"), - RUNNING("running"), - SUCCEEDED("succeeded"), - FAILED("failed"); + PENDING, + RUNNING, + SUCCEEDED, + FAILED + } - private final String value; + /** Explicit migration eligibility and safe blocker for the deployment screen. */ + public record MigrationCapability(boolean allowed, SetupErrorCode blockedBy) { - VerificationState(String value) { - this.value = value; + public MigrationCapability { + MigrationContractValidator.validateCapability(allowed, blockedBy); } - @Override - public String value() { - return value; + public static MigrationCapability permitted() { + return new MigrationCapability(true, null); + } + + public static MigrationCapability blocked(SetupErrorCode blocker) { + return new MigrationCapability(false, blocker); } } @@ -98,10 +149,31 @@ public final class DeploymentApiContract { public record DeploymentView( @NotNull Instant observedAt, @NotNull @Valid ManagementDatabaseSummary managementDatabase, - @NotNull @Valid TelemetryStoreSummary telemetryStore, + @NotNull @Valid TelemetryStoreSummary greptimeDatabase, @NotNull ApplyMode applyMode, - boolean maintenanceMode, - boolean migrationAllowed) { + @NotNull MaintenanceMode maintenanceMode, + @NotNull DeploymentTopology topology, + @NotNull @Valid MigrationCapability migration) { + + public DeploymentView { + MigrationContractValidator.validateDeployment( + managementDatabase, maintenanceMode, topology, migration); + } + } + + /** External target validation input, separate from first-install setup validation. */ + public record MetadataMigrationValidationRequest( + @NotNull MigrationTarget target, + @NotNull @Valid MetadataDatabaseConfiguration targetDatabase) { + + public MetadataMigrationValidationRequest { + MigrationContractValidator.validateTarget(target, targetDatabase); + } + + @Override + public String toString() { + return "MetadataMigrationValidationRequest[target=" + target + ", targetDatabase=]"; + } } /** H2-to-external-database migration input. */ @@ -111,44 +183,64 @@ public final class DeploymentApiContract { @NotNull ApplyMode applyMode) { public MetadataMigrationRequest { - if (target == null || targetDatabase == null || target.databaseKind() != targetDatabase.kind()) { - throw new IllegalArgumentException("Migration target and target database kind must match"); + MigrationContractValidator.validateTarget(target, targetDatabase); + } + + @Override + public String toString() { + return "MetadataMigrationRequest[target=" + target + ", targetDatabase=, applyMode=" + + applyMode + "]"; + } + } + + /** One-shot external-apply export input; target credentials are never retained by the operation. */ + public record MigrationExportRequest( + @NotNull ExportFormat format, + @NotNull MigrationOperationState expectedState, + @NotNull @Valid MetadataDatabaseConfiguration targetDatabase) { + + public MigrationExportRequest { + if (expectedState != MigrationOperationState.AWAITING_EXTERNAL_APPLY + || targetDatabase == null || targetDatabase.kind() == MetadataDatabaseKind.H2) { + throw new IllegalArgumentException("Migration export requires an external target awaiting apply"); } } + + @Override + public String toString() { + return "MigrationExportRequest[format=" + format + ", expectedState=" + expectedState + + ", targetDatabase=]"; + } } /** Safe migration operation view; table identities and verification details are intentionally absent. */ public record MigrationView( @NotBlank String operationId, - @NotNull SetupOperationState state, + @NotNull MigrationOperationState state, @NotNull MetadataDatabaseKind source, @NotNull MigrationTarget target, - @NotNull SetupPhase phase, + @NotNull MigrationStage stage, + @Min(0) @Max(100) int progressPercent, @NotNull Instant createdAt, Instant startedAt, Instant completedAt, - @PositiveOrZero long tablesTotal, - @PositiveOrZero long tablesCopied, @NotNull VerificationState verificationState, SetupErrorCode errorCode, + @PositiveOrZero long nextPollAfterMillis, boolean activationAvailable, + boolean restartRequired, boolean externalApplyRequired) { public MigrationView { - if (source != MetadataDatabaseKind.H2) { - throw new IllegalArgumentException("Migration source must be H2"); - } - if (phase != SetupPhase.MIGRATION_IN_PROGRESS) { - throw new IllegalArgumentException("Migration view must report migration in progress"); - } - if (tablesTotal < 0 || tablesCopied < 0 || tablesCopied > tablesTotal) { - throw new IllegalArgumentException("Migration table counts are inconsistent"); - } + MigrationContractValidator.validateMigration(operationId, source, target, state, stage, progressPercent, + createdAt, startedAt, completedAt, verificationState, errorCode, nextPollAfterMillis, + activationAvailable, restartRequired, externalApplyRequired); } } /** Explicit migration activation input. */ public record ActivateMigrationRequest( - @NotNull SetupOperationState expectedState) { + @NotNull MigrationOperationState expectedState) { } + } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentController.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentController.java new file mode 100644 index 0000000000..e23e1c8d72 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentController.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import jakarta.validation.Valid; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.ActivateMigrationRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentView; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MetadataMigrationRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MetadataMigrationValidationRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationExportRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationView; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationResponse; +import org.apache.hertzbeat.manager.setup.workflow.MigrationExportRenderer; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; + +/** Transport-only adapter that fails safely until the migration workflow is available. */ +@RestController +public final class DeploymentController { + + private final ObjectProvider workflowProvider; + private final ObjectProvider rendererProvider; + + public DeploymentController( + ObjectProvider workflowProvider, + ObjectProvider rendererProvider) { + this.workflowProvider = workflowProvider; + this.rendererProvider = rendererProvider; + } + + @GetMapping(DeploymentApiContract.DEPLOYMENT_PATH) + public ResponseEntity deployment() { + return SetupHttpContract.noStore().body(workflow().deployment()); + } + + @PostMapping(DeploymentApiContract.VALIDATE_PATH) + public ResponseEntity validate( + @Valid @RequestBody MetadataMigrationValidationRequest request) { + return SetupHttpContract.noStore().body(workflow().validate(request)); + } + + @PostMapping(DeploymentApiContract.MIGRATION_PATH) + public ResponseEntity migrate(@Valid @RequestBody MetadataMigrationRequest request) { + return SetupHttpContract.noStore().body(workflow().migrate(request)); + } + + @GetMapping(DeploymentApiContract.MIGRATION_OPERATION_PATH) + public ResponseEntity migration(@PathVariable String operationId) { + MigrationView migration = workflow().migration(operationId); + if (migration == null) { + throw new SetupApiException(SetupApiContract.SetupErrorCode.OPERATION_NOT_FOUND, HttpStatus.NOT_FOUND); + } + return SetupHttpContract.noStore().body(migration); + } + + @PostMapping(DeploymentApiContract.ACTIVATE_PATH) + public ResponseEntity activate( + @PathVariable String operationId, @Valid @RequestBody ActivateMigrationRequest request) { + return SetupHttpContract.noStore().body(workflow().activate(operationId, request)); + } + + @PostMapping(DeploymentApiContract.EXPORT_PATH) + public ResponseEntity export( + @PathVariable String operationId, @Valid @RequestBody MigrationExportRequest request) { + MigrationExportRenderer renderer = renderer(); + ExportResponse metadata = workflow().prepareExport(operationId, request); + StreamingResponseBody body = output -> renderer.write(operationId, request, output); + return SetupHttpContract.noStore() + .header(HttpHeaders.CONTENT_DISPOSITION, + "attachment; filename=\"" + metadata.fileName() + "\"") + .header(HttpHeaders.CONTENT_TYPE, metadata.mediaType()).body(body); + } + + private DeploymentWorkflow workflow() { + DeploymentWorkflow workflow = workflowProvider.getIfUnique(); + if (workflow == null) { + throw unavailable(); + } + return workflow; + } + + private MigrationExportRenderer renderer() { + MigrationExportRenderer renderer = rendererProvider.getIfUnique(); + if (renderer == null) { + throw unavailable(); + } + return renderer; + } + + private SetupApiException unavailable() { + return new SetupApiException( + SetupApiContract.SetupErrorCode.MIGRATION_UNAVAILABLE, HttpStatus.SERVICE_UNAVAILABLE); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentWorkflow.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentWorkflow.java index bce6c4b459..4bf48b1ece 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentWorkflow.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentWorkflow.java @@ -20,8 +20,10 @@ package org.apache.hertzbeat.manager.setup.api; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.ActivateMigrationRequest; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentView; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MetadataMigrationRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MetadataMigrationValidationRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationExportRequest; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationView; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidateRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationResponse; /** Authenticated deployment boundary implemented by a later migration engine milestone. */ @@ -29,11 +31,13 @@ public interface DeploymentWorkflow { DeploymentView deployment(); - ValidationResponse validate(ValidateRequest request); + ValidationResponse validate(MetadataMigrationValidationRequest request); MigrationView migrate(MetadataMigrationRequest request); MigrationView migration(String operationId); MigrationView activate(String operationId, ActivateMigrationRequest request); + + ExportResponse prepareExport(String operationId, MigrationExportRequest request); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/MigrationContractValidator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/MigrationContractValidator.java new file mode 100644 index 0000000000..d0b3aeff57 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/MigrationContractValidator.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import java.time.Instant; +import java.util.Set; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentTopology; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MaintenanceMode; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationCapability; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ManagementDatabaseSummary; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Cross-field invariants for secret-free deployment and migration projections. */ +final class MigrationContractValidator { + + private static final Set OPERATION_ERRORS = Set.of( + SetupErrorCode.MIGRATION_COPY_FAILED, + SetupErrorCode.MIGRATION_VERIFICATION_FAILED, + SetupErrorCode.MIGRATION_ACTIVATION_FAILED, + SetupErrorCode.RESTART_FAILED); + private static final Set CAPABILITY_BLOCKERS = Set.of( + SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED, + SetupErrorCode.MIGRATION_MULTI_NODE_UNSUPPORTED, + SetupErrorCode.MIGRATION_TOPOLOGY_UNAVAILABLE, + SetupErrorCode.MIGRATION_MAINTENANCE_REQUIRED); + + private MigrationContractValidator() { + } + + static void validateCapability(boolean allowed, SetupErrorCode blockedBy) { + if (allowed != (blockedBy == null) + || (blockedBy != null && !CAPABILITY_BLOCKERS.contains(blockedBy))) { + invalid("Migration capability and blocker are inconsistent"); + } + } + + static void validateTarget(MigrationTarget target, MetadataDatabaseConfiguration database) { + if (target == null || database == null || target.databaseKind() != database.kind()) { + invalid("Migration target and target database kind must match"); + } + } + + static void validateDeployment( + ManagementDatabaseSummary database, MaintenanceMode maintenance, + DeploymentTopology topology, MigrationCapability capability) { + if (database == null || maintenance == null || topology == null || capability == null) { + invalid("Deployment migration context is incomplete"); + } + if (database.kind() != MetadataDatabaseKind.H2) { + requireBlocker(capability, SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED); + } else if (topology == DeploymentTopology.MULTI_NODE) { + requireBlocker(capability, SetupErrorCode.MIGRATION_MULTI_NODE_UNSUPPORTED); + } else if (topology == DeploymentTopology.UNKNOWN) { + requireBlocker(capability, SetupErrorCode.MIGRATION_TOPOLOGY_UNAVAILABLE); + } else if (maintenance == MaintenanceMode.INACTIVE) { + requireBlocker(capability, SetupErrorCode.MIGRATION_MAINTENANCE_REQUIRED); + } else if (!capability.allowed()) { + invalid("Active single-node H2 migration must be permitted"); + } + } + + static void validateMigration( + String operationId, MetadataDatabaseKind source, MigrationTarget target, + MigrationOperationState state, MigrationStage stage, + int progress, Instant createdAt, Instant startedAt, Instant completedAt, + VerificationState verification, SetupErrorCode errorCode, long pollAfterMillis, + boolean activationAvailable, boolean restartRequired, boolean externalApplyRequired) { + if (operationId == null || operationId.isBlank() || source != MetadataDatabaseKind.H2 || target == null + || state == null || stage == null + || createdAt == null || verification == null || progress < 0 || progress > 100 + || pollAfterMillis < 0) { + invalid("Migration projection is incomplete or out of range"); + } + validateTimes(state, createdAt, startedAt, completedAt); + validateState(state, stage, progress, verification, errorCode, pollAfterMillis); + validateOutcome(state, errorCode, activationAvailable, restartRequired, externalApplyRequired); + } + + private static void validateTimes( + MigrationOperationState state, Instant createdAt, Instant startedAt, Instant completedAt) { + boolean pending = state == MigrationOperationState.PENDING; + boolean terminal = terminal(state); + if (pending != (startedAt == null) || terminal != (completedAt != null)) { + invalid("Migration timestamps do not match lifecycle state"); + } + if (startedAt != null && startedAt.isBefore(createdAt) + || completedAt != null && completedAt.isBefore(startedAt)) { + invalid("Migration timestamps are out of order"); + } + } + + private static void validateState( + MigrationOperationState state, MigrationStage stage, int progress, + VerificationState verification, SetupErrorCode errorCode, long pollAfterMillis) { + boolean valid = switch (state) { + case PENDING -> stage == MigrationStage.QUEUED && progress == 0 + && verification == VerificationState.PENDING && pollAfterMillis > 0; + case RUNNING -> running(stage, progress, verification) && pollAfterMillis > 0; + case READY_TO_ACTIVATE -> stage == MigrationStage.READY_TO_ACTIVATE && progress == 100 + && verification == VerificationState.SUCCEEDED && pollAfterMillis == 0; + case AWAITING_EXTERNAL_APPLY -> stage == MigrationStage.AWAITING_EXTERNAL_APPLY && progress == 100 + && verification == VerificationState.SUCCEEDED && pollAfterMillis == 0; + case AWAITING_RESTART -> stage == MigrationStage.AWAITING_RESTART && progress == 100 + && verification == VerificationState.SUCCEEDED && pollAfterMillis > 0; + case SUCCEEDED -> stage == MigrationStage.COMPLETED && progress == 100 + && verification == VerificationState.SUCCEEDED && pollAfterMillis == 0; + case FAILED -> stage == MigrationStage.FAILED && failureMatches(errorCode, verification, progress) + && pollAfterMillis == 0; + case ROLLED_BACK -> stage == MigrationStage.ROLLED_BACK + && failureMatches(errorCode, verification, progress) + && pollAfterMillis == 0; + }; + if (!valid) { + invalid("Migration state, stage, progress, verification, or polling is inconsistent"); + } + } + + private static boolean failureMatches( + SetupErrorCode errorCode, VerificationState verification, int progress) { + if (errorCode == null) { + return false; + } + return switch (errorCode) { + case MIGRATION_COPY_FAILED -> verification == VerificationState.PENDING && progress < 100; + case MIGRATION_VERIFICATION_FAILED -> verification == VerificationState.FAILED && progress == 100; + case MIGRATION_ACTIVATION_FAILED, RESTART_FAILED -> + verification == VerificationState.SUCCEEDED && progress == 100; + default -> false; + }; + } + + private static boolean running(MigrationStage stage, int progress, VerificationState verification) { + return switch (stage) { + case COPYING -> progress < 100 && verification == VerificationState.PENDING; + case VERIFYING -> progress == 100 && verification == VerificationState.RUNNING; + case ACTIVATING -> progress == 100 && verification == VerificationState.SUCCEEDED; + case ROLLING_BACK -> verification == VerificationState.SUCCEEDED + || verification == VerificationState.FAILED; + default -> false; + }; + } + + private static void validateOutcome( + MigrationOperationState state, SetupErrorCode errorCode, + boolean activationAvailable, boolean restartRequired, boolean externalApplyRequired) { + boolean failure = state == MigrationOperationState.FAILED || state == MigrationOperationState.ROLLED_BACK; + boolean activatable = state == MigrationOperationState.READY_TO_ACTIVATE; + if (failure != (errorCode != null) || errorCode != null && !OPERATION_ERRORS.contains(errorCode) + || activationAvailable != activatable + || restartRequired != (state == MigrationOperationState.AWAITING_RESTART) + || externalApplyRequired != (state == MigrationOperationState.AWAITING_EXTERNAL_APPLY)) { + invalid("Migration outcome and transition flags are inconsistent"); + } + } + + private static boolean terminal(MigrationOperationState state) { + return state == MigrationOperationState.SUCCEEDED + || state == MigrationOperationState.FAILED + || state == MigrationOperationState.ROLLED_BACK; + } + + private static void requireBlocker(MigrationCapability capability, SetupErrorCode expected) { + if (capability.allowed() || capability.blockedBy() != expected) { + invalid("Deployment migration blocker does not match its structure"); + } + } + + private static void invalid(String message) { + throw new IllegalArgumentException(message); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java index c4253504e6..e440e1c958 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupApiContract.java @@ -254,8 +254,12 @@ public final class SetupApiContract { MIGRATION_SOURCE_UNSUPPORTED("migration_source_unsupported"), MIGRATION_TARGET_NOT_EMPTY("migration_target_not_empty"), MIGRATION_MULTI_NODE_UNSUPPORTED("migration_multi_node_unsupported"), + MIGRATION_TOPOLOGY_UNAVAILABLE("migration_topology_unavailable"), + MIGRATION_MAINTENANCE_REQUIRED("migration_maintenance_required"), + MIGRATION_UNAVAILABLE("migration_unavailable"), MIGRATION_COPY_FAILED("migration_copy_failed"), MIGRATION_VERIFICATION_FAILED("migration_verification_failed"), + MIGRATION_ACTIVATION_NOT_AVAILABLE("migration_activation_not_available"), MIGRATION_ACTIVATION_FAILED("migration_activation_failed"), RESTART_FAILED("restart_failed"); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandler.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandler.java index 80b1405323..7645cad2d7 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandler.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandler.java @@ -32,7 +32,7 @@ import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.RestControllerAdvice; /** Owns safe HTTP classification for setup failures. */ -@RestControllerAdvice(assignableTypes = SetupController.class) +@RestControllerAdvice(assignableTypes = {SetupController.class, DeploymentController.class}) public class SetupExceptionHandler { private static final Logger LOGGER = LoggerFactory.getLogger(SetupExceptionHandler.class); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationPolicy.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationPolicy.java new file mode 100644 index 0000000000..557040ec23 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationPolicy.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentTopology; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentView; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MaintenanceMode; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationView; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.TargetInspection; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiException; +import org.springframework.http.HttpStatus; + +/** Stable migration admission rules consumed by a later copy-engine implementation. */ +public final class MetadataMigrationPolicy { + + public void requireMigrationAllowed( + DeploymentView deployment, MigrationTarget target, TargetInspection targetInspection) { + if (deployment == null || target == null || targetInspection == null) { + throw new SetupApiException(SetupErrorCode.INVALID_REQUEST, HttpStatus.BAD_REQUEST); + } + if (deployment.managementDatabase().kind() != MetadataDatabaseKind.H2) { + throw conflict(SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED); + } + if (deployment.topology() == DeploymentTopology.UNKNOWN) { + throw conflict(SetupErrorCode.MIGRATION_TOPOLOGY_UNAVAILABLE); + } + if (deployment.topology() == DeploymentTopology.MULTI_NODE) { + throw conflict(SetupErrorCode.MIGRATION_MULTI_NODE_UNSUPPORTED); + } + if (deployment.maintenanceMode() != MaintenanceMode.ACTIVE) { + throw conflict(SetupErrorCode.MIGRATION_MAINTENANCE_REQUIRED); + } + if (targetInspection == TargetInspection.UNKNOWN) { + throw conflict(SetupErrorCode.METADATA_CONNECTION_FAILED); + } + if (targetInspection == TargetInspection.NON_EMPTY) { + throw conflict(SetupErrorCode.MIGRATION_TARGET_NOT_EMPTY); + } + } + + public void requireActivationAllowed( + MigrationView operation, MigrationOperationState expectedState) { + if (operation == null) { + throw new SetupApiException(SetupErrorCode.OPERATION_NOT_FOUND, HttpStatus.NOT_FOUND); + } + if (operation.state() != expectedState) { + throw conflict(SetupErrorCode.OPERATION_CONFLICT); + } + if (operation.state() != MigrationOperationState.READY_TO_ACTIVATE) { + throw conflict(SetupErrorCode.MIGRATION_ACTIVATION_NOT_AVAILABLE); + } + } + + private SetupApiException conflict(SetupErrorCode code) { + return new SetupApiException(code, HttpStatus.CONFLICT); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationExportRenderer.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationExportRenderer.java new file mode 100644 index 0000000000..fe1f6bd143 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationExportRenderer.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.IOException; +import java.io.OutputStream; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationExportRequest; + +/** Streaming port for one-shot external-apply content; implementations must not retain request secrets. */ +@FunctionalInterface +public interface MigrationExportRenderer { + + void write(String operationId, MigrationExportRequest request, OutputStream output) throws IOException; +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/DeploymentRouteAuthorizationConfigTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/DeploymentRouteAuthorizationConfigTest.java new file mode 100644 index 0000000000..aa9f61f360 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/DeploymentRouteAuthorizationConfigTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.config; + +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.function.Executable; + +/** Shipped deployment routes require an authenticated administrator. */ +class DeploymentRouteAuthorizationConfigTest { + + private static final List ADMIN_RULES = List.of( + " - /api/config/deployment===get===[admin]", + " - /api/config/deployment/validate===post===[admin]", + " - /api/config/deployment/metadata-migrations===post===[admin]", + " - /api/config/deployment/metadata-migrations/*===get===[admin]", + " - /api/config/deployment/metadata-migrations/*/activate===post===[admin]", + " - /api/config/deployment/metadata-migrations/*/export===post===[admin]"); + private static final List SURENESS_CONFIGS = List.of( + "hertzbeat-startup/src/main/resources/sureness.yml", + "hertzbeat-manager/src/test/resources/sureness.yml", + "hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/sureness.yml", + "script/sureness.yml", + "script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml", + "script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml", + "script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml", + "script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml", + "script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml"); + + @Test + void everyShippedConfigUsesExactAdminRulesAndNoPublicBypass() { + List checks = new ArrayList<>(); + for (String config : SURENESS_CONFIGS) { + checks.add(() -> assertRules(config)); + } + assertAll(checks); + } + + private static void assertRules(String config) throws IOException { + List lines = Files.readAllLines(repoRoot().resolve(config)); + for (String rule : ADMIN_RULES) { + assertTrue(lines.contains(rule), () -> config + " must contain " + rule.trim()); + } + assertFalse(lines.stream().anyMatch(line -> line.contains("/api/config/deployment") + && line.endsWith("===*")), + () -> config + " must not publicly exclude deployment routes"); + } + + private static Path repoRoot() { + Path current = Paths.get("").toAbsolutePath(); + while (current != null && !Files.exists(current.resolve("hertzbeat-manager/pom.xml"))) { + current = current.getParent(); + } + if (current == null) { + throw new IllegalStateException("Cannot locate HertzBeat repository root"); + } + return current; + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContractTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContractTest.java index 64f667d930..d7d9333f4c 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContractTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContractTest.java @@ -18,18 +18,32 @@ package org.apache.hertzbeat.manager.setup.api; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.ObjectMapper; import java.lang.reflect.RecordComponent; +import java.time.Instant; import java.util.Arrays; import java.util.List; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentTopology; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MaintenanceMode; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationCapability; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigSource; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ManagementDatabaseSummary; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreSummary; import org.junit.jupiter.api.Test; /** Freezes authenticated deployment and H2 migration contracts. */ @@ -46,19 +60,71 @@ class DeploymentApiContractTest { DeploymentApiContract.MIGRATION_OPERATION_PATH); assertEquals("/api/config/deployment/metadata-migrations/{operationId}/activate", DeploymentApiContract.ACTIVATE_PATH); + assertEquals("/api/config/deployment/metadata-migrations/{operationId}/export", + DeploymentApiContract.EXPORT_PATH); assertComponents(DeploymentApiContract.DeploymentView.class, "observedAt", "managementDatabase", - "telemetryStore", "applyMode", "maintenanceMode", "migrationAllowed"); + "greptimeDatabase", "applyMode", "maintenanceMode", "topology", "migration"); + assertComponents(DeploymentApiContract.MigrationCapability.class, "allowed", "blockedBy"); + assertComponents(DeploymentApiContract.MetadataMigrationValidationRequest.class, + "target", "targetDatabase"); assertComponents(DeploymentApiContract.MetadataMigrationRequest.class, "target", "targetDatabase", "applyMode"); assertComponents(DeploymentApiContract.MigrationView.class, "operationId", "state", "source", "target", - "phase", "createdAt", "startedAt", "completedAt", "tablesTotal", "tablesCopied", - "verificationState", "errorCode", "activationAvailable", "externalApplyRequired"); + "stage", "progressPercent", "createdAt", "startedAt", "completedAt", "verificationState", + "errorCode", "nextPollAfterMillis", "activationAvailable", "restartRequired", + "externalApplyRequired"); assertComponents(DeploymentApiContract.ActivateMigrationRequest.class, "expectedState"); + assertComponents(DeploymentApiContract.MigrationExportRequest.class, + "format", "expectedState", "targetDatabase"); + assertWireValues(MaintenanceMode.values(), "inactive", "active"); + assertWireValues(DeploymentTopology.values(), "single_node", "multi_node", "unknown"); assertWireValues(MigrationTarget.values(), "mysql", "postgresql"); + assertWireValues(MigrationStage.values(), "queued", "copying", "verifying", "ready_to_activate", + "awaiting_external_apply", "activating", "awaiting_restart", "completed", "rolling_back", + "rolled_back", "failed"); assertWireValues(VerificationState.values(), "pending", "running", "succeeded", "failed"); + assertWireValues(MigrationOperationState.values(), "pending", "running", "ready_to_activate", + "awaiting_external_apply", "awaiting_restart", "succeeded", "failed", "rolled_back"); assertEquals(DeploymentApiContract.MigrationView.class, DeploymentWorkflow.class.getMethod( "activate", String.class, DeploymentApiContract.ActivateMigrationRequest.class) .getReturnType()); + assertEquals(SetupApiContract.ValidationResponse.class, + DeploymentWorkflow.class.getMethod( + "validate", DeploymentApiContract.MetadataMigrationValidationRequest.class) + .getReturnType()); + } + + @Test + void deploymentViewExplainsMigrationAvailabilityWithoutConnectionDetails() throws Exception { + DeploymentApiContract.DeploymentView view = new DeploymentApiContract.DeploymentView( + Instant.parse("2026-08-09T00:00:00Z"), + new ManagementDatabaseSummary(MetadataDatabaseKind.H2, true, ConfigSource.UI_MANAGED, false), + new TelemetryStoreSummary(TelemetryStoreKind.GREPTIME, true, ConfigSource.UI_MANAGED, false), + ApplyMode.MANAGED_WRITE, MaintenanceMode.ACTIVE, DeploymentTopology.SINGLE_NODE, + MigrationCapability.permitted()); + + assertTrue(view.migration().allowed()); + assertNull(view.migration().blockedBy()); + String json = objectMapper.writeValueAsString(view); + assertFalse(json.contains("jdbc:")); + assertFalse(json.contains("password")); + assertFalse(json.contains("table")); + assertThrows(IllegalArgumentException.class, + () -> new MigrationCapability(false, null)); + assertThrows(IllegalArgumentException.class, + () -> new MigrationCapability(true, SetupErrorCode.MIGRATION_MULTI_NODE_UNSUPPORTED)); + assertDeploymentRejected(MetadataDatabaseKind.MYSQL, DeploymentTopology.SINGLE_NODE, + MigrationCapability.permitted()); + assertDeploymentRejected(MetadataDatabaseKind.H2, DeploymentTopology.MULTI_NODE, + MigrationCapability.blocked(SetupErrorCode.MIGRATION_TOPOLOGY_UNAVAILABLE)); + assertDeploymentRejected(MetadataDatabaseKind.H2, DeploymentTopology.UNKNOWN, + MigrationCapability.blocked(SetupErrorCode.MIGRATION_MULTI_NODE_UNSUPPORTED)); + assertThrows(IllegalArgumentException.class, + () -> MigrationCapability.blocked(SetupErrorCode.CONFIG_READ_ONLY)); + assertDeploymentRejected(MetadataDatabaseKind.H2, MaintenanceMode.INACTIVE, + DeploymentTopology.SINGLE_NODE, MigrationCapability.permitted()); + assertDoesNotThrow(() -> deployment(MaintenanceMode.INACTIVE, + MigrationCapability.blocked(SetupErrorCode.MIGRATION_MAINTENANCE_REQUIRED))); } @Test @@ -86,6 +152,127 @@ class DeploymentApiContractTest { MigrationTarget.MYSQL, mysql, ApplyMode.MANAGED_WRITE); assertFalse(objectMapper.writeValueAsString(request).contains(secret)); assertFalse(request.toString().contains(secret)); + DeploymentApiContract.MigrationExportRequest export = new DeploymentApiContract.MigrationExportRequest( + SetupApiContract.ExportFormat.ENV, MigrationOperationState.AWAITING_EXTERNAL_APPLY, mysql); + assertFalse(objectMapper.writeValueAsString(export).contains(secret)); + assertFalse(export.toString().contains(secret)); + } + + @Test + void migrationViewMakesPollingAndActivationTransitionsExplicit() { + DeploymentApiContract.MigrationView ready = migrationView( + MigrationOperationState.READY_TO_ACTIVATE, MigrationStage.READY_TO_ACTIVATE, 100, + VerificationState.SUCCEEDED, null, 0, true, false, false, null); + assertTrue(ready.activationAvailable()); + DeploymentApiContract.MigrationView external = migrationView( + MigrationOperationState.AWAITING_EXTERNAL_APPLY, MigrationStage.AWAITING_EXTERNAL_APPLY, 100, + VerificationState.SUCCEEDED, null, 0, false, false, true, null); + assertFalse(external.activationAvailable()); + assertTrue(external.externalApplyRequired()); + assertThrows(IllegalArgumentException.class, () -> migrationView( + MigrationOperationState.READY_TO_ACTIVATE, MigrationStage.READY_TO_ACTIVATE, 50, + VerificationState.PENDING, null, 0, true, false, false, null)); + assertThrows(IllegalArgumentException.class, () -> migrationView( + MigrationOperationState.READY_TO_ACTIVATE, MigrationStage.COPYING, 100, + VerificationState.SUCCEEDED, null, 0, true, false, false, null)); + assertThrows(IllegalArgumentException.class, () -> migrationView( + MigrationOperationState.SUCCEEDED, MigrationStage.COMPLETED, 100, + VerificationState.SUCCEEDED, Instant.parse("2026-08-08T23:59:59Z"), 0, + false, false, false, null)); + assertThrows(IllegalArgumentException.class, () -> migrationView( + MigrationOperationState.SUCCEEDED, MigrationStage.COMPLETED, 100, + VerificationState.SUCCEEDED, Instant.parse("2026-08-09T00:02:00Z"), 500, + false, false, false, null)); + assertThrows(IllegalArgumentException.class, () -> migrationView( + MigrationOperationState.RUNNING, MigrationStage.COPYING, 45, + VerificationState.PENDING, null, 500, false, false, false, + SetupErrorCode.MIGRATION_COPY_FAILED)); + assertThrows(IllegalArgumentException.class, () -> migrationView( + MigrationOperationState.FAILED, MigrationStage.FAILED, 45, + VerificationState.FAILED, null, 0, false, false, false, + SetupErrorCode.MIGRATION_COPY_FAILED)); + assertEquals(SetupErrorCode.MIGRATION_COPY_FAILED, migrationView( + MigrationOperationState.FAILED, MigrationStage.FAILED, 45, + VerificationState.PENDING, Instant.parse("2026-08-09T00:02:00Z"), 0, + false, false, false, SetupErrorCode.MIGRATION_COPY_FAILED).errorCode()); + assertEquals(VerificationState.FAILED, failedView( + SetupErrorCode.MIGRATION_VERIFICATION_FAILED, VerificationState.FAILED, 100).verificationState()); + assertEquals(VerificationState.SUCCEEDED, failedView( + SetupErrorCode.MIGRATION_ACTIVATION_FAILED, VerificationState.SUCCEEDED, 100).verificationState()); + assertEquals(VerificationState.SUCCEEDED, failedView( + SetupErrorCode.RESTART_FAILED, VerificationState.SUCCEEDED, 100).verificationState()); + assertThrows(IllegalArgumentException.class, () -> failedView( + SetupErrorCode.MIGRATION_COPY_FAILED, VerificationState.FAILED, 45)); + assertThrows(IllegalArgumentException.class, () -> failedView( + SetupErrorCode.MIGRATION_VERIFICATION_FAILED, VerificationState.SUCCEEDED, 100)); + assertThrows(IllegalArgumentException.class, () -> failedView( + SetupErrorCode.MIGRATION_COPY_FAILED, VerificationState.PENDING, 100)); + assertThrows(IllegalArgumentException.class, () -> failedView( + SetupErrorCode.MIGRATION_VERIFICATION_FAILED, VerificationState.FAILED, 99)); + assertEquals(VerificationState.PENDING, rolledBackView( + SetupErrorCode.MIGRATION_COPY_FAILED, VerificationState.PENDING, 99).verificationState()); + assertEquals(VerificationState.SUCCEEDED, rolledBackView( + SetupErrorCode.RESTART_FAILED, VerificationState.SUCCEEDED, 100).verificationState()); + assertThrows(IllegalArgumentException.class, () -> rolledBackView( + SetupErrorCode.MIGRATION_COPY_FAILED, VerificationState.FAILED, 45)); + assertThrows(IllegalArgumentException.class, () -> rolledBackView( + SetupErrorCode.MIGRATION_ACTIVATION_FAILED, VerificationState.SUCCEEDED, 99)); + assertThrows(IllegalArgumentException.class, () -> migrationViewWithIdentity(" ", MigrationTarget.MYSQL)); + assertThrows(IllegalArgumentException.class, () -> migrationViewWithIdentity("migration-1", null)); + } + + private DeploymentApiContract.MigrationView migrationView( + MigrationOperationState state, MigrationStage stage, int progress, + VerificationState verification, Instant completedAt, long nextPollAfterMillis, + boolean activationAvailable, + boolean restartRequired, boolean externalApplyRequired, SetupErrorCode errorCode) { + return new DeploymentApiContract.MigrationView("migration-1", state, MetadataDatabaseKind.H2, + MigrationTarget.MYSQL, stage, progress, Instant.parse("2026-08-09T00:00:00Z"), + Instant.parse("2026-08-09T00:00:01Z"), completedAt, verification, errorCode, nextPollAfterMillis, + activationAvailable, restartRequired, externalApplyRequired); + } + + private DeploymentApiContract.MigrationView failedView( + SetupErrorCode errorCode, VerificationState verification, int progress) { + return migrationView(MigrationOperationState.FAILED, MigrationStage.FAILED, progress, verification, + Instant.parse("2026-08-09T00:02:00Z"), 0, false, false, false, errorCode); + } + + private DeploymentApiContract.MigrationView rolledBackView( + SetupErrorCode errorCode, VerificationState verification, int progress) { + return migrationView(MigrationOperationState.ROLLED_BACK, MigrationStage.ROLLED_BACK, progress, verification, + Instant.parse("2026-08-09T00:02:00Z"), 0, false, false, false, errorCode); + } + + private DeploymentApiContract.MigrationView migrationViewWithIdentity( + String operationId, MigrationTarget target) { + return new DeploymentApiContract.MigrationView(operationId, MigrationOperationState.READY_TO_ACTIVATE, + MetadataDatabaseKind.H2, target, MigrationStage.READY_TO_ACTIVATE, 100, + Instant.parse("2026-08-09T00:00:00Z"), Instant.parse("2026-08-09T00:00:01Z"), null, + VerificationState.SUCCEEDED, null, 0, true, false, false); + } + + private void assertDeploymentRejected( + MetadataDatabaseKind kind, DeploymentTopology topology, MigrationCapability capability) { + assertDeploymentRejected(kind, MaintenanceMode.INACTIVE, topology, capability); + } + + private void assertDeploymentRejected( + MetadataDatabaseKind kind, MaintenanceMode maintenance, + DeploymentTopology topology, MigrationCapability capability) { + assertThrows(IllegalArgumentException.class, () -> new DeploymentApiContract.DeploymentView( + Instant.parse("2026-08-09T00:00:00Z"), + new ManagementDatabaseSummary(kind, true, ConfigSource.UI_MANAGED, false), + new TelemetryStoreSummary(TelemetryStoreKind.GREPTIME, true, ConfigSource.UI_MANAGED, false), + ApplyMode.EXTERNAL_APPLY, maintenance, topology, capability)); + } + + private DeploymentApiContract.DeploymentView deployment( + MaintenanceMode maintenance, MigrationCapability capability) { + return new DeploymentApiContract.DeploymentView(Instant.parse("2026-08-09T00:00:00Z"), + new ManagementDatabaseSummary(MetadataDatabaseKind.H2, true, ConfigSource.UI_MANAGED, false), + new TelemetryStoreSummary(TelemetryStoreKind.GREPTIME, true, ConfigSource.UI_MANAGED, false), + ApplyMode.MANAGED_WRITE, maintenance, DeploymentTopology.SINGLE_NODE, capability); } private void assertComponents(Class type, String... names) { diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerRegistrationTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerRegistrationTest.java new file mode 100644 index 0000000000..f3a35c42ad --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerRegistrationTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; + +import org.apache.hertzbeat.manager.setup.workflow.MigrationExportRenderer; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; +import org.springframework.http.HttpStatus; + +/** Proves real component scanning is independent of optional workflow bean definition order. */ +class DeploymentControllerRegistrationTest { + + private final ApplicationContextRunner context = new ApplicationContextRunner() + .withUserConfiguration(ControllerScan.class); + + @Test + void componentScanAlwaysRegistersSafeTransportWhenDependenciesAreUnavailable() { + context.run(result -> { + assertThat(result).hasSingleBean(DeploymentController.class); + SetupApiException failure = assertThrows(SetupApiException.class, + () -> result.getBean(DeploymentController.class).deployment()); + assertEquals(HttpStatus.SERVICE_UNAVAILABLE, failure.status()); + assertEquals(SetupApiContract.SetupErrorCode.MIGRATION_UNAVAILABLE, failure.errorCode()); + }); + } + + @Test + void componentScanResolvesWorkflowAndRendererRegisteredAlongsideController() { + context.withBean(DeploymentWorkflow.class, () -> mock(DeploymentWorkflow.class)) + .withBean(MigrationExportRenderer.class, () -> mock(MigrationExportRenderer.class)) + .run(result -> assertThat(result).hasSingleBean(DeploymentController.class)); + } + + @Configuration(proxyBeanMethods = false) + @ComponentScan( + basePackageClasses = DeploymentController.class, + useDefaultFilters = false, + includeFilters = @ComponentScan.Filter( + type = FilterType.ASSIGNABLE_TYPE, classes = DeploymentController.class)) + static class ControllerScan { + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerTest.java new file mode 100644 index 0000000000..f9e77982b6 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerTest.java @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.asyncDispatch; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.request; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.io.ByteArrayOutputStream; +import java.time.Instant; +import java.util.List; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationExportRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationView; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportFormat; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationResponse; +import org.apache.hertzbeat.manager.setup.workflow.MigrationExportRenderer; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.support.StaticListableBeanFactory; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; + +/** Transport proof for authenticated, no-store deployment routes. */ +class DeploymentControllerTest { + + private final DeploymentWorkflow workflow = mock(DeploymentWorkflow.class); + private final MigrationExportRenderer exportRenderer = mock(MigrationExportRenderer.class); + private ObjectProvider workflowProvider; + private ObjectProvider rendererProvider; + private MockMvc mvc; + + @BeforeEach + void setUp() { + StaticListableBeanFactory factory = providerFactory(List.of(workflow), List.of(exportRenderer)); + workflowProvider = factory.getBeanProvider(DeploymentWorkflow.class); + rendererProvider = factory.getBeanProvider(MigrationExportRenderer.class); + mvc = mvc(workflowProvider, rendererProvider); + } + + @Test + void routesValidationCreationPollingAndActivationWithoutStoringResponses() throws Exception { + when(workflow.validate(any())).thenReturn(new ValidationResponse( + true, Instant.parse("2026-08-09T00:00:00Z"), null, List.of())); + when(workflow.migrate(any())).thenReturn(readyMigration()); + when(workflow.migration("migration-1")).thenReturn(readyMigration()); + when(workflow.activate(eq("migration-1"), any())).thenReturn(restartingMigration()); + String target = """ + {"target":"mysql","targetDatabase":{"kind":"mysql", + "jdbcUrl":"jdbc:mysql://db/hertzbeat","username":"operator","password":"request-secret"}} + """; + String migration = """ + {"target":"mysql","targetDatabase":{"kind":"mysql", + "jdbcUrl":"jdbc:mysql://db/hertzbeat","username":"operator","password":"request-secret"}, + "applyMode":"managed_write"} + """; + + mvc.perform(post(DeploymentApiContract.VALIDATE_PATH).contentType(MediaType.APPLICATION_JSON) + .content(target)) + .andExpect(status().isOk()).andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.valid").value(true)) + .andExpect(content().string(org.hamcrest.Matchers.not( + org.hamcrest.Matchers.containsString("request-secret")))); + mvc.perform(post(DeploymentApiContract.MIGRATION_PATH).contentType(MediaType.APPLICATION_JSON) + .content(migration)) + .andExpect(status().isOk()).andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.activationAvailable").value(true)); + mvc.perform(get(DeploymentApiContract.MIGRATION_OPERATION_PATH, "migration-1")) + .andExpect(status().isOk()).andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.stage").value("ready_to_activate")); + mvc.perform(post(DeploymentApiContract.ACTIVATE_PATH, "migration-1") + .contentType(MediaType.APPLICATION_JSON) + .content("{\"expectedState\":\"ready_to_activate\"}")) + .andExpect(status().isOk()).andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.restartRequired").value(true)); + + verify(workflow).migration("migration-1"); + } + + @Test + void unexpectedFailuresExposeOnlyStableNoStoreEnvelope() throws Exception { + when(workflow.migration("migration-1")) + .thenThrow(new IllegalStateException("SELECT password FROM internal_table request-secret")); + + mvc.perform(get(DeploymentApiContract.MIGRATION_OPERATION_PATH, "migration-1")) + .andExpect(status().isInternalServerError()) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.errorCode").value("internal_error")) + .andExpect(content().string(org.hamcrest.Matchers.not( + org.hamcrest.Matchers.containsString("request-secret")))) + .andExpect(content().string(org.hamcrest.Matchers.not( + org.hamcrest.Matchers.containsString("internal_table")))); + } + + @Test + void missingMigrationPollIsAnExplicitNotFound() throws Exception { + when(workflow.migration("missing")).thenReturn(null); + + mvc.perform(get(DeploymentApiContract.MIGRATION_OPERATION_PATH, "missing")) + .andExpect(status().isNotFound()) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.errorCode").value("operation_not_found")); + } + + @Test + void missingWorkflowReturnsStableNoStoreUnavailable() throws Exception { + assertDeploymentUnavailable(mvc(List.of(), List.of(exportRenderer))); + } + + @Test + void ambiguousWorkflowReturnsStableNoStoreUnavailable() throws Exception { + assertDeploymentUnavailable(mvc( + List.of(workflow, mock(DeploymentWorkflow.class)), List.of(exportRenderer))); + } + + @Test + void missingOrAmbiguousRendererReturnsStableNoStoreUnavailable() throws Exception { + assertExportUnavailable(mvc(List.of(workflow), List.of())); + assertExportUnavailable(mvc(List.of(workflow), + List.of(exportRenderer, mock(MigrationExportRenderer.class)))); + } + + @Test + void externalApplyExportStreamsOnlyAfterNoStoreAttachmentIsPrepared() throws Exception { + when(workflow.prepareExport(eq("migration-1"), any())).thenReturn( + new ExportResponse("hertzbeat-migration.env", "text/plain")); + String request = """ + {"format":"env","expectedState":"awaiting_external_apply", + "targetDatabase":{"kind":"mysql","jdbcUrl":"jdbc:mysql://db/hertzbeat", + "username":"operator","password":"export-secret"}} + """; + + MvcResult pending = mvc.perform(post(DeploymentApiContract.EXPORT_PATH, "migration-1") + .contentType(MediaType.APPLICATION_JSON).content(request)) + .andExpect(request().asyncStarted()).andReturn(); + mvc.perform(asyncDispatch(pending)) + .andExpect(status().isOk()) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(header().string("Content-Disposition", + "attachment; filename=\"hertzbeat-migration.env\"")); + verify(exportRenderer).write(eq("migration-1"), any(), any()); + } + + @Test + void exportBodyIsDeferredUntilTheStreamingCallbackRuns() throws Exception { + MigrationExportRequest request = new MigrationExportRequest(ExportFormat.ENV, + MigrationOperationState.AWAITING_EXTERNAL_APPLY, + new MetadataDatabaseConfiguration(MetadataDatabaseKind.MYSQL, + "jdbc:mysql://db/hertzbeat", "operator", "export-secret")); + when(workflow.prepareExport("migration-1", request)).thenReturn( + new ExportResponse("hertzbeat-migration.env", "text/plain")); + DeploymentController controller = new DeploymentController(workflowProvider, rendererProvider); + + ResponseEntity response = controller.export("migration-1", request); + + verifyNoInteractions(exportRenderer); + response.getBody().writeTo(new ByteArrayOutputStream()); + verify(exportRenderer).write(eq("migration-1"), eq(request), any()); + } + + private MigrationView readyMigration() { + return new MigrationView("migration-1", MigrationOperationState.READY_TO_ACTIVATE, + SetupApiContract.MetadataDatabaseKind.H2, MigrationTarget.MYSQL, + MigrationStage.READY_TO_ACTIVATE, 100, Instant.parse("2026-08-09T00:00:00Z"), + Instant.parse("2026-08-09T00:00:01Z"), null, + VerificationState.SUCCEEDED, null, 0, true, false, false); + } + + private MigrationView restartingMigration() { + return new MigrationView("migration-1", MigrationOperationState.AWAITING_RESTART, + SetupApiContract.MetadataDatabaseKind.H2, MigrationTarget.MYSQL, + MigrationStage.AWAITING_RESTART, 100, Instant.parse("2026-08-09T00:00:00Z"), + Instant.parse("2026-08-09T00:00:01Z"), null, + VerificationState.SUCCEEDED, null, 1000, false, true, false); + } + + private MockMvc mvc( + List workflows, List renderers) { + StaticListableBeanFactory factory = providerFactory(workflows, renderers); + return mvc(factory.getBeanProvider(DeploymentWorkflow.class), + factory.getBeanProvider(MigrationExportRenderer.class)); + } + + private MockMvc mvc( + ObjectProvider workflows, + ObjectProvider renderers) { + return MockMvcBuilders.standaloneSetup(new DeploymentController(workflows, renderers)) + .setControllerAdvice(new SetupExceptionHandler()).build(); + } + + private StaticListableBeanFactory providerFactory( + List workflows, List renderers) { + StaticListableBeanFactory factory = new StaticListableBeanFactory(); + for (int index = 0; index < workflows.size(); index++) { + factory.addBean("workflow-" + index, workflows.get(index)); + } + for (int index = 0; index < renderers.size(); index++) { + factory.addBean("renderer-" + index, renderers.get(index)); + } + return factory; + } + + private void assertDeploymentUnavailable(MockMvc candidate) throws Exception { + candidate.perform(get(DeploymentApiContract.DEPLOYMENT_PATH)) + .andExpect(status().isServiceUnavailable()) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.errorCode").value("migration_unavailable")); + } + + private void assertExportUnavailable(MockMvc candidate) throws Exception { + String request = """ + {"format":"env","expectedState":"awaiting_external_apply", + "targetDatabase":{"kind":"mysql","jdbcUrl":"jdbc:mysql://db/hertzbeat", + "username":"operator","password":"export-secret"}} + """; + candidate.perform(post(DeploymentApiContract.EXPORT_PATH, "migration-1") + .contentType(MediaType.APPLICATION_JSON).content(request)) + .andExpect(status().isServiceUnavailable()) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.errorCode").value("migration_unavailable")); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationPolicyTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationPolicyTest.java new file mode 100644 index 0000000000..2af6410c5d --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationPolicyTest.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.time.Instant; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentTopology; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentView; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MaintenanceMode; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationCapability; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationView; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.TargetInspection; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigSource; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ManagementDatabaseSummary; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreSummary; +import org.apache.hertzbeat.manager.setup.api.SetupApiException; +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpStatus; + +/** Stable admission and optimistic activation errors independent of a future copy engine. */ +class MetadataMigrationPolicyTest { + + private final MetadataMigrationPolicy policy = new MetadataMigrationPolicy(); + + @Test + void permitsOnlySingleNodeEmptyTargetMigrationFromH2() { + assertDoesNotThrow(() -> policy.requireMigrationAllowed( + deployment(MaintenanceMode.ACTIVE, DeploymentTopology.SINGLE_NODE), + MigrationTarget.MYSQL, TargetInspection.EMPTY)); + assertFailure(SetupErrorCode.MIGRATION_MAINTENANCE_REQUIRED, + () -> policy.requireMigrationAllowed( + deployment(MaintenanceMode.INACTIVE, DeploymentTopology.SINGLE_NODE), + MigrationTarget.MYSQL, TargetInspection.EMPTY)); + assertFailure(SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED, + () -> policy.requireMigrationAllowed( + deployment(MetadataDatabaseKind.MYSQL, MaintenanceMode.ACTIVE, + DeploymentTopology.SINGLE_NODE), + MigrationTarget.POSTGRESQL, TargetInspection.EMPTY)); + assertFailure(SetupErrorCode.MIGRATION_TOPOLOGY_UNAVAILABLE, + () -> policy.requireMigrationAllowed( + deployment(MaintenanceMode.ACTIVE, DeploymentTopology.UNKNOWN), + MigrationTarget.MYSQL, TargetInspection.EMPTY)); + assertFailure(SetupErrorCode.MIGRATION_TARGET_NOT_EMPTY, + () -> policy.requireMigrationAllowed( + deployment(MaintenanceMode.ACTIVE, DeploymentTopology.SINGLE_NODE), + MigrationTarget.MYSQL, TargetInspection.NON_EMPTY)); + assertFailure(SetupErrorCode.METADATA_CONNECTION_FAILED, + () -> policy.requireMigrationAllowed( + deployment(MaintenanceMode.ACTIVE, DeploymentTopology.SINGLE_NODE), + MigrationTarget.MYSQL, TargetInspection.UNKNOWN)); + assertFailure(SetupErrorCode.MIGRATION_MULTI_NODE_UNSUPPORTED, + () -> policy.requireMigrationAllowed( + deployment(MaintenanceMode.ACTIVE, DeploymentTopology.MULTI_NODE), + MigrationTarget.POSTGRESQL, TargetInspection.EMPTY)); + } + + @Test + void activationUsesExpectedStateAndStableLookupAndPhaseErrors() { + assertFailure(HttpStatus.NOT_FOUND, SetupErrorCode.OPERATION_NOT_FOUND, + () -> policy.requireActivationAllowed(null, MigrationOperationState.READY_TO_ACTIVATE)); + assertFailure(SetupErrorCode.OPERATION_CONFLICT, + () -> policy.requireActivationAllowed(readyMigration(), MigrationOperationState.RUNNING)); + assertFailure(SetupErrorCode.MIGRATION_ACTIVATION_NOT_AVAILABLE, + () -> policy.requireActivationAllowed(runningMigration(), MigrationOperationState.RUNNING)); + assertFailure(SetupErrorCode.MIGRATION_ACTIVATION_NOT_AVAILABLE, + () -> policy.requireActivationAllowed( + externalMigration(), MigrationOperationState.AWAITING_EXTERNAL_APPLY)); + assertDoesNotThrow(() -> policy.requireActivationAllowed( + readyMigration(), MigrationOperationState.READY_TO_ACTIVATE)); + } + + private DeploymentView deployment(MaintenanceMode maintenance, DeploymentTopology topology) { + return deployment(MetadataDatabaseKind.H2, maintenance, topology); + } + + private DeploymentView deployment( + MetadataDatabaseKind kind, MaintenanceMode maintenance, DeploymentTopology topology) { + SetupErrorCode blocker = switch (topology) { + case MULTI_NODE -> SetupErrorCode.MIGRATION_MULTI_NODE_UNSUPPORTED; + case UNKNOWN -> SetupErrorCode.MIGRATION_TOPOLOGY_UNAVAILABLE; + case SINGLE_NODE -> kind == MetadataDatabaseKind.H2 ? null : SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED; + }; + if (blocker == null && maintenance == MaintenanceMode.INACTIVE) { + blocker = SetupErrorCode.MIGRATION_MAINTENANCE_REQUIRED; + } + MigrationCapability capability = blocker == null + ? MigrationCapability.permitted() : MigrationCapability.blocked(blocker); + return new DeploymentView(Instant.parse("2026-08-09T00:00:00Z"), + new ManagementDatabaseSummary(kind, true, ConfigSource.UI_MANAGED, false), + new TelemetryStoreSummary(TelemetryStoreKind.GREPTIME, true, ConfigSource.UI_MANAGED, false), + ApplyMode.MANAGED_WRITE, maintenance, topology, capability); + } + + private MigrationView readyMigration() { + return migration(MigrationOperationState.READY_TO_ACTIVATE, MigrationStage.READY_TO_ACTIVATE, + 100, VerificationState.SUCCEEDED, 0, true); + } + + private MigrationView runningMigration() { + return migration(MigrationOperationState.RUNNING, MigrationStage.COPYING, + 25, VerificationState.PENDING, 500, false); + } + + private MigrationView externalMigration() { + return new MigrationView("migration-1", MigrationOperationState.AWAITING_EXTERNAL_APPLY, + MetadataDatabaseKind.H2, MigrationTarget.MYSQL, MigrationStage.AWAITING_EXTERNAL_APPLY, + 100, Instant.parse("2026-08-09T00:00:00Z"), Instant.parse("2026-08-09T00:00:01Z"), + null, VerificationState.SUCCEEDED, null, 0, false, false, true); + } + + private MigrationView migration(MigrationOperationState state, MigrationStage stage, + int progress, VerificationState verification, long poll, boolean activation) { + return new MigrationView("migration-1", state, MetadataDatabaseKind.H2, MigrationTarget.MYSQL, + stage, progress, Instant.parse("2026-08-09T00:00:00Z"), + Instant.parse("2026-08-09T00:00:01Z"), null, verification, null, poll, + activation, false, false); + } + + private void assertFailure(SetupErrorCode expected, Runnable invocation) { + assertFailure(HttpStatus.CONFLICT, expected, invocation); + } + + private void assertFailure(HttpStatus status, SetupErrorCode expected, Runnable invocation) { + SetupApiException failure = assertThrows(SetupApiException.class, invocation::run); + assertEquals(status, failure.status()); + assertEquals(expected, failure.errorCode()); + assertEquals(expected.value(), failure.getMessage()); + } +} diff --git a/hertzbeat-manager/src/test/resources/sureness.yml b/hertzbeat-manager/src/test/resources/sureness.yml index fa6e8d0a95..7f89072b37 100644 --- a/hertzbeat-manager/src/test/resources/sureness.yml +++ b/hertzbeat-manager/src/test/resources/sureness.yml @@ -76,6 +76,12 @@ resourceRole: - /api/notice/**===post===[admin,user] - /api/notice/**===put===[admin,user] - /api/notice/**===delete===[admin] + - /api/config/deployment===get===[admin] + - /api/config/deployment/validate===post===[admin] + - /api/config/deployment/metadata-migrations===post===[admin] + - /api/config/deployment/metadata-migrations/*===get===[admin] + - /api/config/deployment/metadata-migrations/*/activate===post===[admin] + - /api/config/deployment/metadata-migrations/*/export===post===[admin] - /api/config/email===get===[admin,user,guest] - /api/config/email===post===[admin] - /api/config/sms===get===[admin,user,guest] diff --git a/hertzbeat-startup/src/main/resources/sureness.yml b/hertzbeat-startup/src/main/resources/sureness.yml index f470a3b75e..b9e9b3fc7f 100644 --- a/hertzbeat-startup/src/main/resources/sureness.yml +++ b/hertzbeat-startup/src/main/resources/sureness.yml @@ -76,6 +76,12 @@ resourceRole: - /api/notice/**===post===[admin,user] - /api/notice/**===put===[admin,user] - /api/notice/**===delete===[admin] + - /api/config/deployment===get===[admin] + - /api/config/deployment/validate===post===[admin] + - /api/config/deployment/metadata-migrations===post===[admin] + - /api/config/deployment/metadata-migrations/*===get===[admin] + - /api/config/deployment/metadata-migrations/*/activate===post===[admin] + - /api/config/deployment/metadata-migrations/*/export===post===[admin] - /api/config/email===get===[admin,user,guest] - /api/config/email===post===[admin] - /api/config/sms===get===[admin,user,guest] diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml b/script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml index 6084607090..464d0fd720 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml @@ -76,6 +76,12 @@ resourceRole: - /api/notice/**===post===[admin,user] - /api/notice/**===put===[admin,user] - /api/notice/**===delete===[admin] + - /api/config/deployment===get===[admin] + - /api/config/deployment/validate===post===[admin] + - /api/config/deployment/metadata-migrations===post===[admin] + - /api/config/deployment/metadata-migrations/*===get===[admin] + - /api/config/deployment/metadata-migrations/*/activate===post===[admin] + - /api/config/deployment/metadata-migrations/*/export===post===[admin] - /api/config/email===get===[admin,user,guest] - /api/config/email===post===[admin] - /api/config/sms===get===[admin,user,guest] diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml b/script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml index 6084607090..464d0fd720 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml @@ -76,6 +76,12 @@ resourceRole: - /api/notice/**===post===[admin,user] - /api/notice/**===put===[admin,user] - /api/notice/**===delete===[admin] + - /api/config/deployment===get===[admin] + - /api/config/deployment/validate===post===[admin] + - /api/config/deployment/metadata-migrations===post===[admin] + - /api/config/deployment/metadata-migrations/*===get===[admin] + - /api/config/deployment/metadata-migrations/*/activate===post===[admin] + - /api/config/deployment/metadata-migrations/*/export===post===[admin] - /api/config/email===get===[admin,user,guest] - /api/config/email===post===[admin] - /api/config/sms===get===[admin,user,guest] diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml index 6084607090..464d0fd720 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml @@ -76,6 +76,12 @@ resourceRole: - /api/notice/**===post===[admin,user] - /api/notice/**===put===[admin,user] - /api/notice/**===delete===[admin] + - /api/config/deployment===get===[admin] + - /api/config/deployment/validate===post===[admin] + - /api/config/deployment/metadata-migrations===post===[admin] + - /api/config/deployment/metadata-migrations/*===get===[admin] + - /api/config/deployment/metadata-migrations/*/activate===post===[admin] + - /api/config/deployment/metadata-migrations/*/export===post===[admin] - /api/config/email===get===[admin,user,guest] - /api/config/email===post===[admin] - /api/config/sms===get===[admin,user,guest] diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml index 6084607090..464d0fd720 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml @@ -76,6 +76,12 @@ resourceRole: - /api/notice/**===post===[admin,user] - /api/notice/**===put===[admin,user] - /api/notice/**===delete===[admin] + - /api/config/deployment===get===[admin] + - /api/config/deployment/validate===post===[admin] + - /api/config/deployment/metadata-migrations===post===[admin] + - /api/config/deployment/metadata-migrations/*===get===[admin] + - /api/config/deployment/metadata-migrations/*/activate===post===[admin] + - /api/config/deployment/metadata-migrations/*/export===post===[admin] - /api/config/email===get===[admin,user,guest] - /api/config/email===post===[admin] - /api/config/sms===get===[admin,user,guest] diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml index 6084607090..464d0fd720 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml @@ -76,6 +76,12 @@ resourceRole: - /api/notice/**===post===[admin,user] - /api/notice/**===put===[admin,user] - /api/notice/**===delete===[admin] + - /api/config/deployment===get===[admin] + - /api/config/deployment/validate===post===[admin] + - /api/config/deployment/metadata-migrations===post===[admin] + - /api/config/deployment/metadata-migrations/*===get===[admin] + - /api/config/deployment/metadata-migrations/*/activate===post===[admin] + - /api/config/deployment/metadata-migrations/*/export===post===[admin] - /api/config/email===get===[admin,user,guest] - /api/config/email===post===[admin] - /api/config/sms===get===[admin,user,guest] diff --git a/script/sureness.yml b/script/sureness.yml index e249b3d817..c07e5ab7ab 100644 --- a/script/sureness.yml +++ b/script/sureness.yml @@ -76,6 +76,12 @@ resourceRole: - /api/notice/**===post===[admin,user] - /api/notice/**===put===[admin,user] - /api/notice/**===delete===[admin] + - /api/config/deployment===get===[admin] + - /api/config/deployment/validate===post===[admin] + - /api/config/deployment/metadata-migrations===post===[admin] + - /api/config/deployment/metadata-migrations/*===get===[admin] + - /api/config/deployment/metadata-migrations/*/activate===post===[admin] + - /api/config/deployment/metadata-migrations/*/export===post===[admin] - /api/config/email===get===[admin,user,guest] - /api/config/email===post===[admin] - /api/config/sms===get===[admin,user,guest] From 293b983aa45090f4e34cd75a22218fe4c6c65741 Mon Sep 17 00:00:00 2001 From: Logic Date: Sun, 9 Aug 2026 20:14:39 +0800 Subject: [PATCH 32/71] Provision current metadata schemas --- .../setup/workflow/FlywaySchemaHistory.java | 147 +++ .../FlywayTargetSchemaProvisioner.java | 167 ++++ .../setup/workflow/JdbcTargetSchemaState.java | 323 +++++++ .../setup/workflow/TargetSchemaBaseline.java | 171 ++++ .../setup/workflow/TargetSchemaContract.java | 86 ++ .../workflow/TargetSchemaProvisioner.java | 26 + .../TargetSchemaProvisioningException.java | 36 + .../TargetSchemaProvisioningFailure.java | 77 ++ .../FlywayTargetSchemaProvisionerTest.java | 294 ++++++ ...TargetSchemaContractCompatibilityTest.java | 139 +++ .../migration/mysql/B206__current_schema.sql | 881 +++++++++++++++++ .../mysql/V200__create_entity_foundation.sql | 12 +- .../V206__normalize_signal_identifier.sql | 21 + .../postgresql/B206__current_schema.sql | 905 ++++++++++++++++++ .../V206__normalize_sop_schedule_enabled.sql | 20 + .../workflow/HistoricalMetadataSchema.java | 81 ++ .../workflow/MetadataSchemaSnapshot.java | 219 +++++ .../MetadataValidationMySqlDialect.java | 37 + .../TargetSchemaBaselineResourceTest.java | 130 +++ .../TargetSchemaProvisionerDatabaseTest.java | 481 ++++++++++ ...tSchemaProvisionerMetadataFailureTest.java | 150 +++ .../db/historical/mysql/V159__schema.sql | 593 ++++++++++++ .../db/historical/postgresql/V159__schema.sql | 573 +++++++++++ 23 files changed, 5563 insertions(+), 6 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywaySchemaHistory.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioner.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcTargetSchemaState.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaBaseline.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaContract.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioner.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioningException.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioningFailure.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisionerTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaContractCompatibilityTest.java create mode 100644 hertzbeat-startup/src/main/resources/db/migration/mysql/B206__current_schema.sql create mode 100644 hertzbeat-startup/src/main/resources/db/migration/mysql/V206__normalize_signal_identifier.sql create mode 100644 hertzbeat-startup/src/main/resources/db/migration/postgresql/B206__current_schema.sql create mode 100644 hertzbeat-startup/src/main/resources/db/migration/postgresql/V206__normalize_sop_schedule_enabled.sql create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HistoricalMetadataSchema.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataSchemaSnapshot.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataValidationMySqlDialect.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaBaselineResourceTest.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisionerDatabaseTest.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisionerMetadataFailureTest.java create mode 100644 hertzbeat-startup/src/test/resources/db/historical/mysql/V159__schema.sql create mode 100644 hertzbeat-startup/src/test/resources/db/historical/postgresql/V159__schema.sql diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywaySchemaHistory.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywaySchemaHistory.java new file mode 100644 index 0000000000..d40378d321 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywaySchemaHistory.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashSet; +import java.util.Locale; +import java.util.Set; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Owns the Flyway-compatible history layout and current-baseline marker. */ +final class FlywaySchemaHistory { + + private static final String TABLE = "flyway_schema_history"; + private final MetadataDatabaseKind kind; + + FlywaySchemaHistory(MetadataDatabaseKind kind) { + this.kind = kind; + } + + boolean isCurrent(Connection connection, TargetSchemaBaseline baseline) throws SQLException { + Set currentTables = currentBaselineTables(connection); + if (!currentTables.contains(TABLE)) { + return false; + } + String sql = "SELECT installed_rank, version, type, script, checksum, success FROM " + TABLE; + try (Statement statement = connection.createStatement(); ResultSet result = statement.executeQuery(sql)) { + if (!result.next()) { + throw unexpectedTargetState(); + } + boolean current = result.getInt("installed_rank") == 1 + && TargetSchemaBaseline.VERSION.equals(result.getString("version")) + && TargetSchemaBaseline.TYPE.equals(result.getString("type")) + && TargetSchemaBaseline.SCRIPT.equals(result.getString("script")) + && baseline.checksum() == result.getInt("checksum") + && !result.wasNull() + && result.getBoolean("success"); + if (!current || result.next() || !currentTables.contains(TargetSchemaContract.TABLE) + || !currentTables.containsAll(baseline.expectedTables()) + || !new TargetSchemaContract(kind).matches(connection, baseline.expectedTables())) { + throw unexpectedTargetState(); + } + return true; + } + } + + void requireEmptyTarget(Connection connection) throws SQLException { + if (!currentCatalogSchemaObjects(connection).isEmpty()) { + throw unexpectedTargetState(); + } + } + + void record( + Connection connection, + TargetSchemaBaseline baseline, + String installedBy, + int executionTimeMillis) throws SQLException { + new TargetSchemaContract(kind).record(connection, baseline.expectedTables()); + try (Statement statement = connection.createStatement()) { + for (String sql : createStatements()) { + statement.execute(sql); + } + } + String insert = "INSERT INTO " + TABLE + + " (installed_rank, version, description, type, script, checksum, installed_by, execution_time, success)" + + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"; + try (PreparedStatement statement = connection.prepareStatement(insert)) { + statement.setInt(1, 1); + statement.setString(2, TargetSchemaBaseline.VERSION); + statement.setString(3, TargetSchemaBaseline.DESCRIPTION); + statement.setString(4, TargetSchemaBaseline.TYPE); + statement.setString(5, TargetSchemaBaseline.SCRIPT); + statement.setInt(6, baseline.checksum()); + statement.setString(7, abbreviate(installedBy, 100)); + statement.setInt(8, executionTimeMillis); + statement.setBoolean(9, true); + statement.executeUpdate(); + } + } + + private String[] createStatements() { + String table = switch (kind) { + case MYSQL -> "CREATE TABLE " + TABLE + " (" + + "installed_rank INT NOT NULL, version VARCHAR(50), description VARCHAR(200) NOT NULL, " + + "type VARCHAR(20) NOT NULL, script VARCHAR(1000) NOT NULL, checksum INT, " + + "installed_by VARCHAR(100) NOT NULL, installed_on TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, " + + "execution_time INT NOT NULL, success BOOL NOT NULL, " + + "CONSTRAINT flyway_schema_history_pk PRIMARY KEY (installed_rank)) ENGINE=InnoDB"; + case POSTGRESQL -> "CREATE TABLE " + TABLE + " (" + + "installed_rank INT NOT NULL, version VARCHAR(50), description VARCHAR(200) NOT NULL, " + + "type VARCHAR(20) NOT NULL, script VARCHAR(1000) NOT NULL, checksum INTEGER, " + + "installed_by VARCHAR(100) NOT NULL, installed_on TIMESTAMP NOT NULL DEFAULT now(), " + + "execution_time INTEGER NOT NULL, success BOOLEAN NOT NULL, " + + "CONSTRAINT flyway_schema_history_pk PRIMARY KEY (installed_rank))"; + case H2 -> throw new IllegalArgumentException("H2 has no external target schema history"); + }; + return new String[]{table, "CREATE INDEX flyway_schema_history_s_idx ON " + TABLE + " (success)"}; + } + + private Set currentBaselineTables(Connection connection) throws SQLException { + return currentCatalogSchemaObjects(connection, new String[]{"TABLE"}); + } + + private Set currentCatalogSchemaObjects(Connection connection) throws SQLException { + return currentCatalogSchemaObjects(connection, null); + } + + private Set currentCatalogSchemaObjects(Connection connection, String[] types) throws SQLException { + DatabaseMetaData metadata = connection.getMetaData(); + String schema = kind == MetadataDatabaseKind.POSTGRESQL ? connection.getSchema() : null; + Set names = new HashSet<>(); + try (ResultSet objects = metadata.getTables(connection.getCatalog(), schema, "%", types)) { + while (objects.next()) { + names.add(objects.getString("TABLE_NAME").toLowerCase(Locale.ROOT)); + } + } + return Set.copyOf(names); + } + + private static String abbreviate(String value, int maximumLength) { + return value.length() <= maximumLength ? value : value.substring(0, maximumLength); + } + + private static SQLException unexpectedTargetState() { + return new SQLException("Target schema is not empty or does not contain the current baseline", "55000"); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioner.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioner.java new file mode 100644 index 0000000000..9d470811b9 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioner.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Objects; +import java.util.concurrent.locks.ReentrantLock; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Applies the static baseline and writes a history row compatible with subsequent standard Flyway runs. */ +public final class FlywayTargetSchemaProvisioner implements TargetSchemaProvisioner { + + // Admission rejects multi-node migration. The lock prevents concurrent work in this JVM, while a failed MySQL DDL + // sequence can leave partial state that deliberately fails the next precondition instead of pretending to resume. + private static final ReentrantLock PROVISIONING_LOCK = new ReentrantLock(); + + @Override + public void provision(MetadataDatabaseConfiguration target) { + Objects.requireNonNull(target, "target"); + MetadataDatabaseKind kind = supportedKind(target.kind()); + PROVISIONING_LOCK.lock(); + try { + provisionLocked(target, kind); + } finally { + PROVISIONING_LOCK.unlock(); + } + } + + private static void provisionLocked(MetadataDatabaseConfiguration target, MetadataDatabaseKind kind) { + Connection connection; + try { + connection = DriverManager.getConnection(target.jdbcUrl(), target.username(), target.password()); + } catch (SQLException exception) { + throw failure(kind, TargetSchemaProvisioningFailure.Phase.CONNECTION, exception); + } + boolean completed = false; + try { + configureTransaction(connection, kind); + provision(connection, target, kind); + commitTransaction(connection, kind); + completed = true; + } catch (TargetSchemaProvisioningException exception) { + rollbackTransaction(connection, kind); + throw exception; + } finally { + if (!completed) { + closeQuietly(connection); + } + } + try { + connection.close(); + } catch (SQLException exception) { + throw failure(kind, TargetSchemaProvisioningFailure.Phase.CLEANUP, exception); + } + } + + private static void configureTransaction(Connection connection, MetadataDatabaseKind kind) { + if (kind == MetadataDatabaseKind.POSTGRESQL) { + try { + connection.setAutoCommit(false); + } catch (SQLException exception) { + throw failure(kind, TargetSchemaProvisioningFailure.Phase.TRANSACTION, exception); + } + } + } + + private static void commitTransaction(Connection connection, MetadataDatabaseKind kind) { + if (kind == MetadataDatabaseKind.POSTGRESQL) { + try { + connection.commit(); + } catch (SQLException exception) { + throw failure(kind, TargetSchemaProvisioningFailure.Phase.TRANSACTION, exception); + } + } + } + + private static void rollbackTransaction(Connection connection, MetadataDatabaseKind kind) { + if (kind == MetadataDatabaseKind.POSTGRESQL) { + try { + connection.rollback(); + } catch (SQLException ignored) { + // Preserve the sanitized failure from the operation phase. + } + } + } + + private static void closeQuietly(Connection connection) { + try { + connection.close(); + } catch (SQLException ignored) { + // Never attach raw driver diagnostics to the sanitized operation failure. + } + } + + private static void provision( + Connection connection, MetadataDatabaseConfiguration target, MetadataDatabaseKind kind) { + TargetSchemaBaseline baseline; + try { + baseline = TargetSchemaBaseline.load(kind); + } catch (IOException exception) { + throw failure(kind, TargetSchemaProvisioningFailure.Phase.BASELINE_RESOURCE, exception); + } + FlywaySchemaHistory history = new FlywaySchemaHistory(kind); + try { + if (history.isCurrent(connection, baseline)) { + return; + } + history.requireEmptyTarget(connection); + } catch (SQLException exception) { + throw failure(kind, TargetSchemaProvisioningFailure.Phase.PRECONDITION, exception); + } + int executionTimeMillis; + try { + executionTimeMillis = execute(connection, baseline); + } catch (SQLException exception) { + throw failure(kind, TargetSchemaProvisioningFailure.Phase.BASELINE_EXECUTION, exception); + } + try { + history.record(connection, baseline, target.username(), executionTimeMillis); + } catch (SQLException exception) { + throw failure(kind, TargetSchemaProvisioningFailure.Phase.HISTORY_WRITE, exception); + } + } + + private static int execute(Connection connection, TargetSchemaBaseline baseline) throws SQLException { + long startedAt = System.nanoTime(); + try (Statement statement = connection.createStatement()) { + for (String sql : baseline.statements()) { + statement.execute(sql); + } + } + return Math.toIntExact(Math.min(Integer.MAX_VALUE, (System.nanoTime() - startedAt) / 1_000_000L)); + } + + private static TargetSchemaProvisioningException failure( + MetadataDatabaseKind kind, TargetSchemaProvisioningFailure.Phase phase, Throwable exception) { + return new TargetSchemaProvisioningException(kind, TargetSchemaProvisioningFailure.from(phase, exception)); + } + + private static MetadataDatabaseKind supportedKind(MetadataDatabaseKind kind) { + return switch (Objects.requireNonNull(kind, "target kind")) { + case MYSQL -> MetadataDatabaseKind.MYSQL; + case POSTGRESQL -> MetadataDatabaseKind.POSTGRESQL; + case H2 -> throw new IllegalArgumentException("External target schema provisioning does not support H2"); + }; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcTargetSchemaState.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcTargetSchemaState.java new file mode 100644 index 0000000000..e8dd12de4d --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcTargetSchemaState.java @@ -0,0 +1,323 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Captures only JDBC metadata that is stable across compatible drivers and vendor versions. */ +final class JdbcTargetSchemaState { + + private JdbcTargetSchemaState() { + } + + static SchemaState capture( + Connection connection, + MetadataDatabaseKind kind, + Set baselineTables) throws SQLException { + DatabaseMetaData metadata = connection.getMetaData(); + String catalog = connection.getCatalog(); + String schema = kind == MetadataDatabaseKind.POSTGRESQL ? connection.getSchema() : null; + FactCollector facts = new FactCollector(); + for (String table : baselineTables.stream().sorted().toList()) { + facts.add("table", table); + readColumns(metadata, catalog, schema, table, kind, facts); + readPrimaryKey(metadata, catalog, schema, table, facts); + readIndexes(metadata, catalog, schema, table, facts); + readForeignKeys(metadata, catalog, schema, table, facts); + } + return facts.build(); + } + + private static void readColumns( + DatabaseMetaData metadata, + String catalog, + String schema, + String table, + MetadataDatabaseKind kind, + FactCollector facts) throws SQLException { + try (ResultSet columns = metadata.getColumns(catalog, schema, table, null)) { + while (columns.next()) { + int jdbcType = columns.getInt("DATA_TYPE"); + int size = columns.getInt("COLUMN_SIZE"); + int scale = columns.getInt("DECIMAL_DIGITS"); + facts.add( + "column", + table, + normalize(columns.getString("COLUMN_NAME")), + stableTypeFamily(kind, jdbcType, size, scale), + nullable(columns.getInt("NULLABLE"))); + } + } + } + + private static void readPrimaryKey( + DatabaseMetaData metadata, + String catalog, + String schema, + String table, + FactCollector facts) throws SQLException { + OrderedColumns columns = new OrderedColumns(); + try (ResultSet keys = metadata.getPrimaryKeys(catalog, schema, table)) { + while (keys.next()) { + columns.add(keys.getShort("KEY_SEQ"), normalize(keys.getString("COLUMN_NAME"))); + } + } + if (!columns.isEmpty()) { + facts.add("primary-key", table, columns.definition()); + } + } + + private static void readIndexes( + DatabaseMetaData metadata, + String catalog, + String schema, + String table, + FactCollector facts) throws SQLException { + Map indexes = new HashMap<>(); + int unnamedIndex = 0; + try (ResultSet rows = metadata.getIndexInfo(catalog, schema, table, false, false)) { + while (rows.next()) { + String name = rows.getString("INDEX_NAME"); + String column = rows.getString("COLUMN_NAME"); + short position = rows.getShort("ORDINAL_POSITION"); + if (column == null || rows.getShort("TYPE") == DatabaseMetaData.tableIndexStatistic) { + continue; + } + // Names group composite rows from one JDBC result only; the semantic fact never retains the name. + if (name == null && position == 1) { + unnamedIndex++; + } + String group = name == null ? "' : normalize(name); + boolean unique = !rows.getBoolean("NON_UNIQUE"); + indexes.computeIfAbsent(group, ignored -> new IndexColumns(unique)) + .add(position, normalize(column)); + } + } + indexes.values().forEach(index -> facts.add( + "index", table, Boolean.toString(index.unique()), index.columns().definition())); + } + + private static void readForeignKeys( + DatabaseMetaData metadata, + String catalog, + String schema, + String table, + FactCollector facts) throws SQLException { + Map keys = new HashMap<>(); + int unnamedKey = 0; + try (ResultSet rows = metadata.getImportedKeys(catalog, schema, table)) { + while (rows.next()) { + String referencedTable = normalize(rows.getString("PKTABLE_NAME")); + String name = rows.getString("FK_NAME"); + short position = rows.getShort("KEY_SEQ"); + String updateRule = foreignKeyRule(rows.getShort("UPDATE_RULE")); + String deleteRule = foreignKeyRule(rows.getShort("DELETE_RULE")); + String deferrability = foreignKeyDeferrability(rows.getShort("DEFERRABILITY")); + // As with indexes, the provider name only groups rows and is absent from the persisted definition. + if (name == null && position == 1) { + unnamedKey++; + } + String group = name == null ? "' : normalize(name); + keys.computeIfAbsent(group, ignored -> + new ForeignKeyColumns(referencedTable, updateRule, deleteRule, deferrability)) + .add( + position, + normalize(rows.getString("FKCOLUMN_NAME")), + normalize(rows.getString("PKCOLUMN_NAME"))); + } + } + keys.values().forEach(key -> facts.add( + "foreign-key", table, key.localColumns(), key.referencedTable(), key.referencedColumns(), + key.updateRule(), key.deleteRule(), key.deferrability())); + } + + private static String stableTypeFamily( + MetadataDatabaseKind kind, + int jdbcType, + int size, + int scale) { + return switch (jdbcType) { + case Types.BOOLEAN -> "boolean"; + case Types.BIT -> kind == MetadataDatabaseKind.MYSQL ? "boolean" : "binary-bit(" + size + ')'; + case Types.TINYINT -> kind == MetadataDatabaseKind.MYSQL && size == 1 ? "boolean" : "tinyint"; + case Types.SMALLINT -> "smallint"; + case Types.INTEGER -> "integer"; + case Types.BIGINT -> "bigint"; + case Types.NUMERIC, Types.DECIMAL -> "decimal(" + size + ',' + scale + ')'; + case Types.REAL -> "real"; + case Types.FLOAT -> "float"; + case Types.DOUBLE -> "double"; + case Types.CHAR, Types.NCHAR, Types.VARCHAR, Types.NVARCHAR -> "character(" + size + ')'; + case Types.LONGVARCHAR, Types.LONGNVARCHAR, Types.CLOB, Types.NCLOB -> "large-text"; + case Types.BINARY, Types.VARBINARY -> "binary(" + size + ')'; + case Types.LONGVARBINARY, Types.BLOB -> "large-binary"; + case Types.DATE -> "date"; + case Types.TIME, Types.TIME_WITH_TIMEZONE -> "time"; + case Types.TIMESTAMP, Types.TIMESTAMP_WITH_TIMEZONE -> "timestamp"; + default -> "jdbc-type(" + jdbcType + ')'; + }; + } + + private static String nullable(int value) throws SQLException { + return switch (value) { + case DatabaseMetaData.columnNoNulls -> "required"; + case DatabaseMetaData.columnNullable -> "nullable"; + default -> throw new SQLException("Target schema column nullability is unknown", "55000"); + }; + } + + private static String foreignKeyRule(short value) throws SQLException { + return switch (value) { + case DatabaseMetaData.importedKeyCascade -> "cascade"; + case DatabaseMetaData.importedKeyRestrict -> "restrict"; + case DatabaseMetaData.importedKeySetNull -> "set-null"; + case DatabaseMetaData.importedKeyNoAction -> "no-action"; + case DatabaseMetaData.importedKeySetDefault -> "set-default"; + default -> throw new SQLException("Target schema foreign-key rule is unknown", "55000"); + }; + } + + private static String foreignKeyDeferrability(short value) throws SQLException { + return switch (value) { + case DatabaseMetaData.importedKeyInitiallyDeferred -> "initially-deferred"; + case DatabaseMetaData.importedKeyInitiallyImmediate -> "initially-immediate"; + case DatabaseMetaData.importedKeyNotDeferrable -> "not-deferrable"; + default -> throw new SQLException("Target schema foreign-key deferrability is unknown", "55000"); + }; + } + + private static String normalize(String value) { + return value == null ? "" : value.toLowerCase(Locale.ROOT); + } + + record SchemaState(Map facts) { + + SchemaState { + facts = Map.copyOf(facts); + } + } + + private static final class FactCollector { + + private final Map facts = new TreeMap<>(); + + void add(String... parts) { + facts.merge(String.join("|", parts), 1, Integer::sum); + } + + SchemaState build() { + return new SchemaState(facts); + } + } + + private static class OrderedColumns { + + private final Map columns = new TreeMap<>(); + + void add(short position, String column) { + columns.put(position, column); + } + + boolean isEmpty() { + return columns.isEmpty(); + } + + String definition() { + return String.join(",", columns.values()); + } + } + + private static final class IndexColumns { + + private final boolean unique; + private final OrderedColumns columns = new OrderedColumns(); + + private IndexColumns(boolean unique) { + this.unique = unique; + } + + void add(short position, String column) { + columns.add(position, column); + } + + boolean unique() { + return unique; + } + + OrderedColumns columns() { + return columns; + } + } + + private static final class ForeignKeyColumns { + + private final String referencedTable; + private final String updateRule; + private final String deleteRule; + private final String deferrability; + private final OrderedColumns localColumns = new OrderedColumns(); + private final OrderedColumns referencedColumns = new OrderedColumns(); + + private ForeignKeyColumns( + String referencedTable, String updateRule, String deleteRule, String deferrability) { + this.referencedTable = referencedTable; + this.updateRule = updateRule; + this.deleteRule = deleteRule; + this.deferrability = deferrability; + } + + void add(short position, String localColumn, String referencedColumn) { + localColumns.add(position, localColumn); + referencedColumns.add(position, referencedColumn); + } + + String localColumns() { + return localColumns.definition(); + } + + String referencedTable() { + return referencedTable; + } + + String referencedColumns() { + return referencedColumns.definition(); + } + + String updateRule() { + return "update=" + updateRule; + } + + String deleteRule() { + return "delete=" + deleteRule; + } + + String deferrability() { + return "deferrability=" + deferrability; + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaBaseline.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaBaseline.java new file mode 100644 index 0000000000..e63684b16e --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaBaseline.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.StringReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.zip.CRC32; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Loaded current-version baseline plus the metadata expected by Flyway schema history. */ +final class TargetSchemaBaseline { + + private static final Pattern CREATE_TABLE = Pattern.compile( + "(?i)^create\\s+table\\s+(?:if\\s+not\\s+exists\\s+)?([a-z][a-z0-9_]*)\\s*\\("); + static final String VERSION = "206"; + static final String DESCRIPTION = "current schema"; + static final String SCRIPT = "B206__current_schema.sql"; + static final String TYPE = "SQL_BASELINE"; + + private final List statements; + private final Set expectedTables; + private final int checksum; + + private TargetSchemaBaseline(List statements, Set expectedTables, int checksum) { + this.statements = statements; + this.expectedTables = expectedTables; + this.checksum = checksum; + } + + static TargetSchemaBaseline load(MetadataDatabaseKind kind) throws IOException { + String vendor = switch (kind) { + case MYSQL -> "mysql"; + case POSTGRESQL -> "postgresql"; + case H2 -> throw new IllegalArgumentException("H2 has no external target baseline"); + }; + String location = "/db/migration/" + vendor + "/" + SCRIPT; + try (InputStream input = TargetSchemaBaseline.class.getResourceAsStream(location)) { + if (input == null) { + throw new IOException("Target schema baseline resource is missing"); + } + String sql = new String(input.readAllBytes(), StandardCharsets.UTF_8); + List statements = splitStatements(sql); + return new TargetSchemaBaseline(statements, expectedTables(statements), checksum(sql)); + } + } + + List statements() { + return statements; + } + + int checksum() { + return checksum; + } + + Set expectedTables() { + return expectedTables; + } + + private static int checksum(String sql) throws IOException { + CRC32 checksum = new CRC32(); + try (BufferedReader reader = new BufferedReader(new StringReader(sql))) { + String line = reader.readLine(); + if (line != null) { + line = removeByteOrderMark(line); + do { + checksum.update(line.getBytes(StandardCharsets.UTF_8)); + } while ((line = reader.readLine()) != null); + } + } + return (int) checksum.getValue(); + } + + private static String removeByteOrderMark(String line) { + return line.startsWith("\ufeff") ? line.substring(1) : line; + } + + private static List splitStatements(String script) throws IOException { + List statements = new ArrayList<>(); + StringBuilder current = new StringBuilder(); + char quote = 0; + boolean lineComment = false; + for (int index = 0; index < script.length(); index++) { + char character = script.charAt(index); + char next = index + 1 < script.length() ? script.charAt(index + 1) : 0; + if (lineComment) { + if (character == '\n' || character == '\r') { + lineComment = false; + current.append(character); + } + continue; + } + if (quote == 0 && character == '-' && next == '-') { + lineComment = true; + index++; + continue; + } + if (quote == 0 && (character == '\'' || character == '"' || character == '`')) { + quote = character; + current.append(character); + continue; + } + if (quote != 0 && character == quote) { + current.append(character); + if (next == quote) { + current.append(next); + index++; + } else { + quote = 0; + } + continue; + } + if (quote == 0 && character == ';') { + addStatement(statements, current); + continue; + } + current.append(character); + } + if (quote != 0) { + throw new IOException("Target schema baseline contains an unterminated quoted value"); + } + addStatement(statements, current); + return List.copyOf(statements); + } + + private static void addStatement(List statements, StringBuilder current) { + String statement = current.toString().trim(); + if (!statement.isEmpty()) { + statements.add(statement); + } + current.setLength(0); + } + + private static Set expectedTables(List statements) throws IOException { + Set tables = new LinkedHashSet<>(); + for (String statement : statements) { + Matcher matcher = CREATE_TABLE.matcher(statement); + if (matcher.find()) { + tables.add(matcher.group(1).toLowerCase(Locale.ROOT)); + } + } + if (tables.isEmpty()) { + throw new IOException("Target schema baseline does not declare any tables"); + } + return Set.copyOf(tables); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaContract.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaContract.java new file mode 100644 index 0000000000..e5514ff611 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaContract.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Persists and compares the human-readable semantic contract for a provisioned baseline. */ +final class TargetSchemaContract { + + static final String TABLE = "flyway_schema_contract"; + private static final String CREATE_TABLE = "CREATE TABLE " + TABLE + " (" + + "contract_id INT NOT NULL, database_kind VARCHAR(20) NOT NULL, definition TEXT NOT NULL, " + + "occurrences INT NOT NULL, CONSTRAINT flyway_schema_contract_pk PRIMARY KEY (contract_id))"; + + private final MetadataDatabaseKind kind; + + TargetSchemaContract(MetadataDatabaseKind kind) { + this.kind = kind; + } + + void record(Connection connection, Set baselineTables) throws SQLException { + JdbcTargetSchemaState.SchemaState state = JdbcTargetSchemaState.capture(connection, kind, baselineTables); + try (Statement statement = connection.createStatement()) { + statement.execute(CREATE_TABLE); + } + String insert = "INSERT INTO " + TABLE + + " (contract_id, database_kind, definition, occurrences) VALUES (?, ?, ?, ?)"; + try (PreparedStatement statement = connection.prepareStatement(insert)) { + int contractId = 1; + for (Map.Entry fact : state.facts().entrySet()) { + statement.setInt(1, contractId++); + statement.setString(2, kind.name()); + statement.setString(3, fact.getKey()); + statement.setInt(4, fact.getValue()); + statement.addBatch(); + } + statement.executeBatch(); + } + } + + boolean matches(Connection connection, Set baselineTables) throws SQLException { + return JdbcTargetSchemaState.capture(connection, kind, baselineTables).equals(readRecordedState(connection)); + } + + private JdbcTargetSchemaState.SchemaState readRecordedState(Connection connection) throws SQLException { + Map facts = new TreeMap<>(); + String select = "SELECT database_kind, definition, occurrences FROM " + TABLE; + try (PreparedStatement statement = connection.prepareStatement(select)) { + try (ResultSet rows = statement.executeQuery()) { + while (rows.next()) { + if (!kind.name().equals(rows.getString("database_kind"))) { + throw new SQLException("Target schema contract contains another database kind", "55000"); + } + String definition = rows.getString("definition"); + if (facts.put(definition, rows.getInt("occurrences")) != null) { + throw new SQLException("Target schema contract contains duplicate definitions", "55000"); + } + } + } + } + return new JdbcTargetSchemaState.SchemaState(facts); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioner.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioner.java new file mode 100644 index 0000000000..a6f17b69ed --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioner.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; + +/** Provisions an empty external metadata target at the current schema version. */ +public interface TargetSchemaProvisioner { + + void provision(MetadataDatabaseConfiguration target); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioningException.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioningException.java new file mode 100644 index 0000000000..395c8543bc --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioningException.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Safe failure boundary that does not retain target credentials, URLs, or baseline SQL. */ +public final class TargetSchemaProvisioningException extends RuntimeException { + + private final TargetSchemaProvisioningFailure failure; + + TargetSchemaProvisioningException( + MetadataDatabaseKind kind, TargetSchemaProvisioningFailure failure) { + super("Target schema provisioning failed for " + kind); + this.failure = failure; + } + + public TargetSchemaProvisioningFailure failure() { + return failure; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioningFailure.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioningFailure.java new file mode 100644 index 0000000000..0ccc6c222d --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioningFailure.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.SQLException; +import java.util.Locale; +import java.util.Objects; +import java.util.regex.Pattern; + +/** Stable failure fields suitable for a durable setup-operation diagnostic. */ +public record TargetSchemaProvisioningFailure( + Phase phase, + String migrationVersion, + String sqlState, + int vendorCode) { + + private static final Pattern SQL_STATE = Pattern.compile("[0-9A-Z]{5}"); + + public TargetSchemaProvisioningFailure { + Objects.requireNonNull(phase, "phase"); + Objects.requireNonNull(migrationVersion, "migrationVersion"); + } + + static TargetSchemaProvisioningFailure from(Phase phase, Throwable exception) { + SQLException sqlException = findSqlException(exception); + return new TargetSchemaProvisioningFailure( + phase, + TargetSchemaBaseline.VERSION, + sqlException == null ? null : sanitizedSqlState(sqlException.getSQLState()), + sqlException == null ? 0 : sqlException.getErrorCode()); + } + + private static String sanitizedSqlState(String sqlState) { + if (sqlState == null) { + return null; + } + String normalized = sqlState.toUpperCase(Locale.ROOT); + return SQL_STATE.matcher(normalized).matches() ? normalized : null; + } + + private static SQLException findSqlException(Throwable exception) { + Throwable current = exception; + for (int depth = 0; current != null && depth < 16; depth++) { + if (current instanceof SQLException sqlException) { + return sqlException; + } + current = current.getCause(); + } + return null; + } + + /** Lifecycle boundary that failed without retaining an exception or SQL text. */ + public enum Phase { + CONNECTION, + BASELINE_RESOURCE, + PRECONDITION, + BASELINE_EXECUTION, + HISTORY_WRITE, + TRANSACTION, + CLEANUP + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisionerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisionerTest.java new file mode 100644 index 0000000000..c9b1dcb6df --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisionerTest.java @@ -0,0 +1,294 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.lang.reflect.Proxy; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.DriverPropertyInfo; +import java.sql.SQLException; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.logging.Logger; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.flywaydb.core.Flyway; +import org.flywaydb.core.api.logging.Log; +import org.flywaydb.core.api.logging.LogCreator; +import org.flywaydb.core.api.logging.LogFactory; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.ResourceLock; + +class FlywayTargetSchemaProvisionerTest { + + private static final String FLYWAY_LOG_FACTORY = "flyway-log-factory"; + + @Test + void rejectsEmbeddedTargetsBeforeConnectionOpen() { + MetadataDatabaseConfiguration target = new MetadataDatabaseConfiguration( + MetadataDatabaseKind.H2, "jdbc:h2:mem:not-opened", "sa", "not-retained"); + + assertThatThrownBy(() -> new FlywayTargetSchemaProvisioner().provision(target)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("External target schema provisioning does not support H2"); + } + + @Test + void failureDoesNotRetainJdbcUrlPasswordOrFlywayDetails() { + String jdbcUrl = "jdbc:mysql://invalid.example.test:3306/hertzbeat"; + String password = "not-retained"; + MetadataDatabaseConfiguration target = + new MetadataDatabaseConfiguration(MetadataDatabaseKind.MYSQL, jdbcUrl, "operator", password); + + assertThatThrownBy(() -> new FlywayTargetSchemaProvisioner().provision(target)) + .isInstanceOf(TargetSchemaProvisioningException.class) + .hasMessage("Target schema provisioning failed for MYSQL") + .hasNoCause() + .message() + .doesNotContain(jdbcUrl, password, "SELECT", "CREATE"); + } + + @Test + void failureExposesOnlyStableStructuredDiagnostics() throws Exception { + String jdbcUrl = "jdbc:diagnostic://private.example.test/hertzbeat?password=secret-value"; + Driver driver = new DiagnosticFailureDriver(jdbcUrl); + DriverManager.registerDriver(driver); + try { + MetadataDatabaseConfiguration target = new MetadataDatabaseConfiguration( + MetadataDatabaseKind.MYSQL, jdbcUrl, "operator", "secret-value"); + + assertThatThrownBy(() -> new FlywayTargetSchemaProvisioner().provision(target)) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, exception -> { + assertThat(exception.failure()).isEqualTo(new TargetSchemaProvisioningFailure( + TargetSchemaProvisioningFailure.Phase.CONNECTION, + "206", + "08006", + 1045)); + assertThat(exception).hasNoCause(); + assertThat(exception.getMessage()) + .doesNotContain(jdbcUrl, "secret-value", "SELECT", "CREATE"); + }); + } finally { + DriverManager.deregisterDriver(driver); + } + } + + @Test + void closeFailureIsNotAttachedToSanitizedOperationFailure() throws Exception { + String jdbcUrl = "jdbc:close-failure://private.example.test/hertzbeat"; + Driver driver = new CloseFailureDriver(jdbcUrl); + DriverManager.registerDriver(driver); + try { + MetadataDatabaseConfiguration target = new MetadataDatabaseConfiguration( + MetadataDatabaseKind.MYSQL, jdbcUrl, "operator", "secret-value"); + + assertThatThrownBy(() -> new FlywayTargetSchemaProvisioner().provision(target)) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, exception -> { + assertThat(exception.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.BASELINE_RESOURCE); + assertThat(exception.getSuppressed()).isEmpty(); + assertThat(exception.getMessage()).doesNotContain(jdbcUrl, "secret-value", "SELECT"); + }); + } finally { + DriverManager.deregisterDriver(driver); + } + } + + @Test + @ResourceLock(FLYWAY_LOG_FACTORY) + void provisioningDoesNotReplaceLoggerUsedByAnInterleavedFlywayOperation() throws Exception { + RecordingLogCreator recording = new RecordingLogCreator(); + LogFactory.setLogCreator(recording); + CountDownLatch provisioningFinished = new CountDownLatch(1); + try (ExecutorService executor = Executors.newFixedThreadPool(2)) { + Future provisioning = executor.submit(() -> { + try { + MetadataDatabaseConfiguration target = new MetadataDatabaseConfiguration( + MetadataDatabaseKind.MYSQL, + "jdbc:mysql://127.0.0.1:1/hertzbeat?connectTimeout=100", + "operator", + "test-only-password"); + assertThatThrownBy(() -> new FlywayTargetSchemaProvisioner().provision(target)) + .isInstanceOf(TargetSchemaProvisioningException.class); + } finally { + provisioningFinished.countDown(); + } + }); + Future interleavedLog = executor.submit(() -> { + provisioningFinished.await(); + LogFactory.getLog(FlywayTargetSchemaProvisionerTest.class).info("unrelated-flyway-operation"); + return null; + }); + + provisioning.get(); + interleavedLog.get(); + assertThat(recording.messages()).contains("unrelated-flyway-operation"); + } finally { + LogFactory.setConfiguration(Flyway.configure()); + } + } + + private static final class RecordingLogCreator implements LogCreator { + + private final List messages = new CopyOnWriteArrayList<>(); + + @Override + public Log createLogger(Class clazz) { + return new RecordingLog(messages); + } + + List messages() { + return List.copyOf(messages); + } + } + + private record RecordingLog(List messages) implements Log { + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public void debug(String message) { + messages.add(message); + } + + @Override + public void info(String message) { + messages.add(message); + } + + @Override + public void warn(String message) { + messages.add(message); + } + + @Override + public void error(String message) { + messages.add(message); + } + + @Override + public void error(String message, Exception exception) { + messages.add(message); + } + + @Override + public void notice(String message) { + messages.add(message); + } + } + + private record DiagnosticFailureDriver(String acceptedUrl) implements Driver { + + @Override + public Connection connect(String url, Properties info) throws SQLException { + if (!acceptsURL(url)) { + return null; + } + throw new SQLException("Connection failed for " + url + " after SELECT secret-value", "08006", 1045); + } + + @Override + public boolean acceptsURL(String url) { + return acceptedUrl.equals(url); + } + + @Override + public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) { + return new DriverPropertyInfo[0]; + } + + @Override + public int getMajorVersion() { + return 1; + } + + @Override + public int getMinorVersion() { + return 0; + } + + @Override + public boolean jdbcCompliant() { + return false; + } + + @Override + public Logger getParentLogger() { + return Logger.getAnonymousLogger(); + } + } + + private record CloseFailureDriver(String acceptedUrl) implements Driver { + + @Override + public Connection connect(String url, Properties info) { + if (!acceptsURL(url)) { + return null; + } + return (Connection) Proxy.newProxyInstance( + getClass().getClassLoader(), new Class[]{Connection.class}, (proxy, method, arguments) -> { + if (method.getName().equals("close")) { + throw new SQLException("close leaked " + url + " after SELECT secret-value", "08006", 999); + } + return null; + }); + } + + @Override + public boolean acceptsURL(String url) { + return acceptedUrl.equals(url); + } + + @Override + public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) { + return new DriverPropertyInfo[0]; + } + + @Override + public int getMajorVersion() { + return 1; + } + + @Override + public int getMinorVersion() { + return 0; + } + + @Override + public boolean jdbcCompliant() { + return false; + } + + @Override + public Logger getParentLogger() { + return Logger.getAnonymousLogger(); + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaContractCompatibilityTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaContractCompatibilityTest.java new file mode 100644 index 0000000000..495f9843a3 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaContractCompatibilityTest.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Set; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.api.Test; + +class TargetSchemaContractCompatibilityTest { + + private static final Set TABLES = Set.of("contract_parent", "contract_child"); + + @Test + void equivalentSchemaDoesNotDependOnJdbcPresentationMetadata() throws Exception { + try (Connection first = schema("representation_a", "alpha", "1", "first", "asc", true); + Connection second = schema("representation_b", "beta", "2", "second", "desc", false)) { + assertThat(JdbcTargetSchemaState.capture(first, MetadataDatabaseKind.MYSQL, TABLES)) + .isEqualTo(JdbcTargetSchemaState.capture(second, MetadataDatabaseKind.MYSQL, TABLES)); + } + } + + @Test + void recordedContractRejectsDuplicateHumanReadableDefinitions() throws Exception { + try (Connection connection = schema("duplicate_contract", "stable", "1", "remarks", "asc", false); + Statement statement = connection.createStatement()) { + TargetSchemaContract contract = new TargetSchemaContract(MetadataDatabaseKind.MYSQL); + contract.record(connection, TABLES); + statement.execute("INSERT INTO flyway_schema_contract " + + "(contract_id, database_kind, definition, occurrences) " + + "SELECT 9999, database_kind, definition, occurrences FROM flyway_schema_contract " + + "FETCH FIRST 1 ROW ONLY"); + + assertThatThrownBy(() -> contract.matches(connection, TABLES)) + .isInstanceOf(SQLException.class) + .hasMessage("Target schema contract contains duplicate definitions"); + } + } + + @Test + void semanticStatePreservesIntegerWidth() throws Exception { + try (Connection baseline = semanticSchema("semantic_baseline", "BIGINT", ""); + Connection narrowerInteger = semanticSchema("semantic_integer", "INTEGER", "")) { + JdbcTargetSchemaState.SchemaState baselineState = + JdbcTargetSchemaState.capture(baseline, MetadataDatabaseKind.MYSQL, TABLES); + + assertThat(JdbcTargetSchemaState.capture(narrowerInteger, MetadataDatabaseKind.MYSQL, TABLES)) + .isNotEqualTo(baselineState); + } + } + + @Test + void semanticStatePreservesForeignKeyActions() throws Exception { + try (Connection baseline = semanticSchema("foreign_key_baseline", "BIGINT", ""); + Connection cascadingDelete = + semanticSchema("foreign_key_cascade", "BIGINT", " ON DELETE CASCADE")) { + JdbcTargetSchemaState.SchemaState baselineState = + JdbcTargetSchemaState.capture(baseline, MetadataDatabaseKind.MYSQL, TABLES); + + assertThat(JdbcTargetSchemaState.capture(cascadingDelete, MetadataDatabaseKind.MYSQL, TABLES)) + .isNotEqualTo(baselineState); + } + } + + @Test + void recordedContractRejectsRowsForAnotherDatabaseKind() throws Exception { + try (Connection connection = schema("cross_kind_contract", "stable", "1", "remarks", "asc", false); + Statement statement = connection.createStatement()) { + TargetSchemaContract contract = new TargetSchemaContract(MetadataDatabaseKind.MYSQL); + contract.record(connection, TABLES); + statement.execute("INSERT INTO flyway_schema_contract " + + "(contract_id, database_kind, definition, occurrences) " + + "VALUES (9999, 'POSTGRESQL', 'table|intruder', 1)"); + + assertThatThrownBy(() -> contract.matches(connection, TABLES)) + .isInstanceOfSatisfying(SQLException.class, + exception -> assertThat(exception.getSQLState()).isEqualTo("55000")); + } + } + + private static Connection schema( + String database, + String objectSuffix, + String defaultValue, + String remarks, + String indexOrder, + boolean identity) throws Exception { + Connection connection = DriverManager.getConnection( + "jdbc:h2:mem:" + database + ";MODE=MySQL;DATABASE_TO_LOWER=TRUE;DB_CLOSE_DELAY=-1", "sa", ""); + try (Statement statement = connection.createStatement()) { + String id = identity ? "BIGINT GENERATED BY DEFAULT AS IDENTITY" : "BIGINT"; + statement.execute("CREATE TABLE contract_parent (id " + id + + ", CONSTRAINT pk_parent_" + objectSuffix + " PRIMARY KEY (id))"); + statement.execute("CREATE TABLE contract_child (id BIGINT NOT NULL, parent_id BIGINT, " + + "label VARCHAR(64) NOT NULL DEFAULT '" + defaultValue + "', " + + "CONSTRAINT pk_child_" + objectSuffix + " PRIMARY KEY (id), " + + "CONSTRAINT fk_child_" + objectSuffix + + " FOREIGN KEY (parent_id) REFERENCES contract_parent(id))"); + statement.execute("COMMENT ON COLUMN contract_child.label IS '" + remarks + "'"); + statement.execute("CREATE INDEX ix_child_" + objectSuffix + + " ON contract_child(parent_id " + indexOrder + ", label " + indexOrder + ")"); + } + return connection; + } + + private static Connection semanticSchema(String database, String integerType, String foreignKeyAction) + throws Exception { + Connection connection = DriverManager.getConnection( + "jdbc:h2:mem:" + database + ";MODE=MySQL;DATABASE_TO_LOWER=TRUE;DB_CLOSE_DELAY=-1", "sa", ""); + try (Statement statement = connection.createStatement()) { + statement.execute("CREATE TABLE contract_parent (id BIGINT PRIMARY KEY)"); + statement.execute("CREATE TABLE contract_child (id BIGINT PRIMARY KEY, parent_id " + integerType + + ", CONSTRAINT fk_child FOREIGN KEY (parent_id) REFERENCES contract_parent(id)" + + foreignKeyAction + ")"); + } + return connection; + } +} diff --git a/hertzbeat-startup/src/main/resources/db/migration/mysql/B206__current_schema.sql b/hertzbeat-startup/src/main/resources/db/migration/mysql/B206__current_schema.sql new file mode 100644 index 0000000000..f5099c33d5 --- /dev/null +++ b/hertzbeat-startup/src/main/resources/db/migration/mysql/B206__current_schema.sql @@ -0,0 +1,881 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0. +-- +-- Static V206 schema baseline for provisioning an empty MySQL target. +-- Future versioned migrations start at V207 or later. + + create table hzb_ai_conversation ( + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + creator varchar(255), + modifier varchar(255), + title varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_ai_message ( + conversation_id bigint, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + creator varchar(255), + modifier varchar(255), + role varchar(255), + content longtext not null, + primary key (id) + ) engine=InnoDB; + + create table hzb_alert_define ( + enable bit not null, + period integer, + times integer, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + datasource varchar(100), + name varchar(100) not null, + expr varchar(2048), + labels varchar(2048), + template varchar(2048), + annotations varchar(4096), + creator varchar(255), + modifier varchar(255), + type varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_alert_define_monitor_bind ( + alert_define_id bigint, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + monitor_id bigint, + primary key (id) + ) engine=InnoDB; + + create table hzb_alert_group ( + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + common_labels varchar(2048), + group_key varchar(2048) character set ascii, + group_labels varchar(2048), + alert_fingerprints TEXT, + common_annotations TEXT, + creator varchar(255), + modifier varchar(255), + status varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_alert_group_converge ( + enable bit, + gmt_create datetime(6), + gmt_update datetime(6), + group_interval bigint, + group_wait bigint, + id bigint not null auto_increment, + repeat_interval bigint, + name varchar(100) not null, + group_labels varchar(1024), + creator varchar(255), + modifier varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_alert_inhibit ( + enable bit, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + name varchar(100) not null, + equal_labels varchar(2048), + source_labels varchar(2048), + target_labels varchar(2048), + creator varchar(255), + modifier varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_alert_silence ( + enable bit not null, + match_all bit not null, + times integer, + type tinyint not null, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + period_end datetime(6), + period_start datetime(6), + name varchar(100) not null, + labels varchar(2048), + creator varchar(255), + days varchar(255), + modifier varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_alert_single ( + trigger_times integer, + active_at bigint, + end_at bigint, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + start_at bigint, + fingerprint varchar(2048) character set ascii, + labels varchar(2048), + annotations varchar(4096), + content varchar(4096), + creator varchar(255), + modifier varchar(255), + status varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_bulletin ( + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + fields varchar(4096), + monitor_ids varchar(4096), + app varchar(255), + creator varchar(255), + modifier varchar(255), + name varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_collector ( + status tinyint not null, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + creator varchar(255), + ip varchar(255) not null, + mode varchar(255), + modifier varchar(255), + name varchar(255) not null, + version varchar(255), + + primary key (id), + check ((status>=0)) + ) engine=InnoDB; + + create table hzb_collector_monitor_bind ( + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + monitor_id bigint, + collector varchar(255), + creator varchar(255), + modifier varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_config ( + gmt_create datetime(6), + gmt_update datetime(6), + + content varchar(8192), + creator varchar(255), + modifier varchar(255), + type varchar(255) not null, + primary key (type) + ) engine=InnoDB; + + create table hzb_define ( + gmt_create datetime(6), + gmt_update datetime(6), + app varchar(255) not null, + creator varchar(255), + modifier varchar(255), + content longtext, + primary key (app) + ) engine=InnoDB; + + create table hzb_grafana_dashboard ( + enabled bit not null, + monitor_id bigint not null, + version bigint, + folder_uid varchar(255), + slug varchar(255), + status varchar(255), + uid varchar(255), + url varchar(255), + primary key (monitor_id) + ) engine=InnoDB; + + create table hzb_history ( + dou float(53), + int32 integer, + metric_type tinyint, + id bigint not null auto_increment, + time bigint, + str varchar(2048), + app varchar(255), + metric_labels varchar(5000), + metric varchar(255), + metrics varchar(255), + instance varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_metrics_favorite ( + create_time datetime(6), + id bigint not null auto_increment, + monitor_id bigint not null, + creator varchar(255) not null, + metrics_name varchar(255) not null, + primary key (id) + ) engine=InnoDB; + + create table hzb_monitor ( + intervals integer, + status tinyint not null, + type tinyint not null, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null, + job_id bigint, + schedule_type varchar(20), + app varchar(100), + cron_expression varchar(100), + instance varchar(100), + name varchar(100), + scrape varchar(100), + annotations varchar(4096), + labels varchar(4096), + creator varchar(255), + description varchar(255), + modifier varchar(255), + primary key (id), + check ((status<=4) and (status>=0)) + ) engine=InnoDB; + + create table hzb_monitor_bind ( + biz_id bigint, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + monitor_id bigint, + creator varchar(255), + key_str varchar(255), + modifier varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_notice_receiver ( + agent_id integer, + lark_receive_type tinyint, + type tinyint not null, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + smn_ak varchar(22), + smn_project_id varchar(32), + smn_region varchar(32), + smn_sk varchar(42), + email varchar(100), + name varchar(100) not null, + phone varchar(100), + access_token varchar(300), + discord_bot_token varchar(300), + discord_channel_id varchar(300), + gotify_token varchar(300), + hook_auth_token varchar(300), + hook_auth_type varchar(300), + server_chan_token varchar(300), + slack_web_hook_url varchar(300), + smn_topic_urn varchar(300), + wechat_id varchar(300), + hook_url varchar(1000), + app_id varchar(255), + app_secret varchar(255), + chat_id varchar(255), + corp_id varchar(255), + creator varchar(255), + modifier varchar(255), + party_id varchar(255), + tag_id varchar(255), + tg_bot_token varchar(255), + tg_message_thread_id varchar(255), + tg_user_id varchar(255), + user_id varchar(255), + primary key (id), + check ((type>=0)) + ) engine=InnoDB; + + create table hzb_notice_rule ( + enable bit not null, + filter_all bit not null, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + period_end datetime(6), + period_start datetime(6), + template_id bigint, + name varchar(100) not null, + template_name varchar(100), + labels varchar(2048), + creator varchar(255), + days varchar(255), + modifier varchar(255), + receiver_id varchar(255) not null, + receiver_name varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_notice_template ( + preset boolean default false, + type tinyint not null, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + name varchar(100) not null, + creator varchar(255), + modifier varchar(255), + content text not null, + primary key (id), + check ((type>=0)) + ) engine=InnoDB; + + create table hzb_param ( + type tinyint not null, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + monitor_id bigint, + field varchar(100) not null, + param_value varchar(8126), + primary key (id), + check ((type>=0)) + ) engine=InnoDB; + + create table hzb_param_define ( + hide bit not null, + param_limit smallint, + required bit not null, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + name varchar(2048), + param_options varchar(2048), + app varchar(255), + creator varchar(255), + default_value varchar(255), + depend varchar(255), + field varchar(255), + key_alias varchar(255), + modifier varchar(255), + param_range varchar(255), + placeholder varchar(255), + type varchar(255), + value_alias varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_plugin_item ( + id bigint not null auto_increment, + metadata_id bigint, + class_identifier varchar(255), + type enum ('POST_ALERT','POST_COLLECT'), + primary key (id) + ) engine=InnoDB; + + create table hzb_plugin_metadata ( + enable_status bit, + param_count integer, + gmt_create datetime(6), + id bigint not null auto_increment, + creator varchar(255), + jar_file_path varchar(255), + name varchar(255) not null, + primary key (id) + ) engine=InnoDB; + + create table hzb_plugin_param ( + type tinyint not null, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + plugin_metadata_id bigint not null, + field varchar(100) not null, + param_value varchar(8126), + primary key (id), + check ((type>=0)) + ) engine=InnoDB; + + create table hzb_push_metrics ( + id bigint not null auto_increment, + monitor_id bigint, + time bigint, + metrics varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_sop_schedule ( + id bigint not null auto_increment, + conversation_id bigint not null comment 'Conversation ID to push results to', + sop_name varchar(64) not null comment 'Name of the SOP skill to execute', + sop_params varchar(1024) comment 'SOP execution parameters in JSON format', + cron_expression varchar(64) not null comment 'Cron expression for scheduling', + enabled tinyint default 1 comment 'Whether the schedule is enabled', + last_run_time datetime comment 'Last execution time', + next_run_time datetime comment 'Next scheduled execution time', + creator varchar(64) comment 'Creator of this record', + modifier varchar(64) comment 'Last modifier', + gmt_create datetime default current_timestamp comment 'Create time', + gmt_update datetime default current_timestamp on update current_timestamp comment 'Update time', + primary key (id) + ) engine=InnoDB; + + create table hzb_status_page_component ( + config_state tinyint not null, + method tinyint not null, + state tinyint not null, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + org_id bigint, + labels varchar(4096), + creator varchar(255), + description varchar(255), + modifier varchar(255), + name varchar(255) not null, + primary key (id) + ) engine=InnoDB; + + create table hzb_status_page_history ( + abnormal integer, + normal integer, + state tinyint not null, + unknowing integer, + uptime float(53), + component_id bigint, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + timestamp bigint, + creator varchar(255), + modifier varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_status_page_incident ( + state tinyint not null, + end_time bigint, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + org_id bigint, + start_time bigint, + creator varchar(255), + modifier varchar(255), + name varchar(255) not null, + primary key (id) + ) engine=InnoDB; + + create table hzb_status_page_incident_component_bind ( + component_id bigint, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + incident_id bigint, + primary key (id) + ) engine=InnoDB; + + create table hzb_status_page_incident_content ( + state tinyint not null, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + incident_id bigint, + timestamp bigint, + creator varchar(255), + message TEXT, + modifier varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_status_page_org ( + state tinyint not null, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + color varchar(255), + creator varchar(255), + description varchar(255) not null, + feedback varchar(255), + home varchar(255) not null, + logo varchar(255) not null, + modifier varchar(255), + name varchar(255) not null, + primary key (id) + ) engine=InnoDB; + + create table hzb_tag ( + type tinyint, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + tag_value varchar(2048), + creator varchar(255), + description varchar(255), + modifier varchar(255), + name varchar(255) not null, + primary key (id), + check ((type<=3) and (type>=0)) + ) engine=InnoDB; + + + create index idx_message_conversation_id + on hzb_ai_message (conversation_id); + + create index idx_alert_define_id + on hzb_alert_define_monitor_bind (alert_define_id); + + create index idx_monitor_id + on hzb_alert_define_monitor_bind (monitor_id); + + alter table hzb_alert_group + add constraint unique_group_key unique (group_key); + + create index idx_name + on hzb_alert_group_converge (name); + + alter table hzb_alert_single + add constraint unique_fingerprint unique (fingerprint); + + alter table hzb_collector + add constraint uk_hzb_collector_name unique (name); + + create index idx_collector_monitor_collector + on hzb_collector_monitor_bind (collector); + + create index idx_collector_monitor_monitor_id + on hzb_collector_monitor_bind (monitor_id); + + + + + + create index idx_hzb_history_instance + on hzb_history (instance); + + create index idx_hzb_history_app + on hzb_history (app); + + create index idx_hzb_history_metrics + on hzb_history (metrics); + + create index idx_hzb_history_metric + on hzb_history (metric); + + alter table hzb_metrics_favorite + add constraint uk_hzb_metrics_favorite unique (creator, monitor_id, metrics_name); + + create index idx_hzb_monitor_app + on hzb_monitor (app); + + create index idx_hzb_monitor_instance + on hzb_monitor (instance); + + create index idx_hzb_monitor_name + on hzb_monitor (name); + + create index index_monitor_bind + on hzb_monitor_bind (biz_id); + + create index index_monitor_bin + on hzb_monitor_bind (monitor_id); + + create index idx_hzb_param_monitor_id + on hzb_param (monitor_id); + + alter table hzb_param + add constraint uk_hzb_param_monitor_field unique (monitor_id, field); + + create index idx_hzb_plugin_param_plugin_metadata_id + on hzb_plugin_param (plugin_metadata_id); + + alter table hzb_plugin_param + add constraint uk_hzb_plugin_param_metadata_field unique (plugin_metadata_id, field); + + create index idx_push_metrics_monitor_id + on hzb_push_metrics (monitor_id); + + create index idx_push_metrics_time + on hzb_push_metrics (time); + + create index idx_schedule_conversation_id + on hzb_sop_schedule (conversation_id); + + create index idx_schedule_enabled_next + on hzb_sop_schedule (enabled, next_run_time); + + create index index_incident_component + on hzb_status_page_incident_component_bind (incident_id); + + create index idx_incident_component_component_id + on hzb_status_page_incident_component_bind (component_id); + + alter table hzb_ai_message + add constraint fk_hzb_ai_message_conversation + foreign key (conversation_id) + references hzb_ai_conversation (id); + + alter table hzb_plugin_item + add constraint fk_hzb_plugin_item_metadata + foreign key (metadata_id) + references hzb_plugin_metadata (id); + + alter table hzb_status_page_incident_content + add constraint fk_hzb_incident_content_incident + foreign key (incident_id) + references hzb_status_page_incident (id); + +CREATE TABLE hzb_entity ( + id BIGINT PRIMARY KEY COMMENT 'Entity ID', + entity_type VARCHAR(32) NOT NULL COMMENT 'Entity type', + name VARCHAR(128) NOT NULL COMMENT 'Entity name', + display_name VARCHAR(128) COMMENT 'Entity display name', + sub_type VARCHAR(128) COMMENT 'Entity subtype from HertzBeat v1 definition', + namespace VARCHAR(128) COMMENT 'Namespace', + environment VARCHAR(128) COMMENT 'Deployment environment', + status VARCHAR(32) NOT NULL COMMENT 'Aggregated entity status', + criticality VARCHAR(32) COMMENT 'Entity criticality', + owner VARCHAR(128) COMMENT 'Entity owner', + additional_owners TEXT COMMENT 'Additional owners json', + runbook VARCHAR(512) COMMENT 'Runbook URL or identifier', + lifecycle VARCHAR(64) COMMENT 'Entity lifecycle', + tier VARCHAR(64) COMMENT 'Entity tier', + system_name VARCHAR(128) COMMENT 'Owning system', + component_of TEXT COMMENT 'Parent components or systems', + components TEXT COMMENT 'Child components that belong to this system', + implemented_by TEXT COMMENT 'ImplementedBy references json', + api_interface TEXT COMMENT 'API interface definition json', + inherit_from VARCHAR(255) COMMENT 'Entity inheritance reference', + languages TEXT COMMENT 'Programming languages json', + links TEXT COMMENT 'Entity links json', + contacts TEXT COMMENT 'Entity contacts json', + integrations TEXT COMMENT 'Entity integrations json', + extensions TEXT COMMENT 'Entity custom extensions json', + hertzbeat TEXT COMMENT 'HertzBeat definition blocks json', + source VARCHAR(32) NOT NULL COMMENT 'Entity source', + description VARCHAR(512) COMMENT 'Entity description', + labels VARCHAR(4096) COMMENT 'Entity labels json', + tags TEXT COMMENT 'Entity catalog tags json', + workspace_id VARCHAR(64) NOT NULL DEFAULT 'default' COMMENT 'Entity workspace boundary', + creator VARCHAR(64) COMMENT 'Creator', + modifier VARCHAR(64) COMMENT 'Modifier', + gmt_create DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time', + gmt_update DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Update time', + INDEX idx_hzb_entity_type (entity_type), + INDEX idx_hzb_entity_status (status), + INDEX idx_hzb_entity_name (name), + INDEX idx_hzb_entity_owner (owner), + INDEX idx_hzb_entity_workspace (workspace_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; + +CREATE TABLE hzb_entity_identity ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + entity_id BIGINT NOT NULL COMMENT 'Entity ID', + identity_type VARCHAR(32) NOT NULL COMMENT 'Identity source type', + identity_key VARCHAR(128) NOT NULL COMMENT 'Identity key', + identity_value VARCHAR(512) NOT NULL COMMENT 'Identity value', + normalized_value VARCHAR(512) NOT NULL COMMENT 'Normalized identity value', + priority INT NOT NULL DEFAULT 40 COMMENT 'Identity priority', + primary_identity TINYINT NOT NULL DEFAULT 0 COMMENT 'Whether primary identity', + creator VARCHAR(64) COMMENT 'Creator', + modifier VARCHAR(64) COMMENT 'Modifier', + gmt_create DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time', + gmt_update DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Update time', + UNIQUE KEY uk_hzb_entity_identity (entity_id, identity_key, normalized_value), + INDEX idx_hzb_entity_identity_lookup (identity_key, normalized_value), + INDEX idx_hzb_entity_identity_entity (entity_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; + +CREATE TABLE hzb_entity_monitor_bind ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + entity_id BIGINT NOT NULL COMMENT 'Entity ID', + monitor_id BIGINT NOT NULL COMMENT 'Monitor ID', + bind_type VARCHAR(32) NOT NULL COMMENT 'Bind type', + bind_source VARCHAR(64) NOT NULL COMMENT 'Bind source', + status VARCHAR(16) NOT NULL COMMENT 'Bind status', + score INT NOT NULL DEFAULT 100 COMMENT 'Bind score', + match_context TEXT COMMENT 'Matched identities json', + creator VARCHAR(64) COMMENT 'Creator', + modifier VARCHAR(64) COMMENT 'Modifier', + gmt_create DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time', + gmt_update DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Update time', + UNIQUE KEY uk_hzb_entity_monitor_bind (entity_id, monitor_id), + INDEX idx_hzb_entity_monitor_bind_entity (entity_id), + INDEX idx_hzb_entity_monitor_bind_monitor (monitor_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; + +CREATE TABLE hzb_entity_relation ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + source_entity_id BIGINT NOT NULL COMMENT 'Source entity ID', + target_entity_id BIGINT NULL COMMENT 'Target entity ID', + target_ref VARCHAR(255) COMMENT 'Target entity reference', + relation_type VARCHAR(32) NOT NULL COMMENT 'Relation type', + relation_source VARCHAR(32) NOT NULL COMMENT 'Relation source', + status VARCHAR(16) NOT NULL COMMENT 'Relation status', + score INT NOT NULL DEFAULT 100 COMMENT 'Relation score', + description VARCHAR(255) COMMENT 'Relation description', + attributes TEXT COMMENT 'Relation attributes json', + creator VARCHAR(64) COMMENT 'Creator', + modifier VARCHAR(64) COMMENT 'Modifier', + gmt_create DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time', + gmt_update DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Update time', + UNIQUE KEY uk_hzb_entity_relation (source_entity_id, target_entity_id, relation_type), + INDEX idx_hzb_entity_relation_source (source_entity_id), + INDEX idx_hzb_entity_relation_target (target_entity_id), + INDEX idx_hzb_entity_relation_target_ref (target_ref) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; + +CREATE TABLE hzb_entity_definition_activity ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + entity_id BIGINT NOT NULL COMMENT 'Entity ID', + workspace_id VARCHAR(64) NOT NULL DEFAULT 'default' COMMENT 'Workspace ID', + activity_type VARCHAR(32) NOT NULL COMMENT 'Definition activity type', + format VARCHAR(16) NOT NULL COMMENT 'Definition format', + status VARCHAR(16) NOT NULL COMMENT 'Activity status', + summary VARCHAR(128) NOT NULL COMMENT 'Activity summary', + detail VARCHAR(255) COMMENT 'Activity detail', + creator VARCHAR(64) COMMENT 'Creator', + gmt_create DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time', + INDEX idx_hzb_entity_definition_activity_entity (entity_id), + INDEX idx_hzb_entity_definition_activity_workspace_time (workspace_id, gmt_create), + INDEX idx_hzb_entity_definition_activity_time (gmt_create) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; + +CREATE TABLE hzb_entity_governance_state ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + state_scope VARCHAR(32) NOT NULL COMMENT 'Governance scope, such as discovery', + state_kind VARCHAR(32) NOT NULL COMMENT 'State kind, such as preset or activity', + workspace_id VARCHAR(64) NOT NULL DEFAULT 'default' COMMENT 'Workspace ID', + state_key VARCHAR(128) NOT NULL COMMENT 'Stable state key', + state_name VARCHAR(128) COMMENT 'State display name', + status VARCHAR(32) COMMENT 'State status', + content TEXT COMMENT 'State JSON content', + creator VARCHAR(64) COMMENT 'Creator', + gmt_create DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time', + gmt_update DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Update time', + UNIQUE KEY uk_hzb_entity_governance_state_scope_kind_workspace_key (state_scope, state_kind, workspace_id, state_key), + INDEX idx_hzb_entity_governance_state_scope_kind (state_scope, state_kind), + INDEX idx_hzb_entity_governance_state_scope_kind_workspace (state_scope, state_kind, workspace_id), + INDEX idx_hzb_entity_governance_state_update (gmt_update), + INDEX idx_hzb_entity_governance_state_creator (creator) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; + +CREATE TABLE hzb_auth_token ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) COMMENT 'API token name', + token_hash VARCHAR(128) NOT NULL COMMENT 'SHA-256 hash of token value', + token_mask VARCHAR(64) COMMENT 'Masked token value for display', + token_scope VARCHAR(32) NOT NULL DEFAULT 'api-admin' COMMENT 'Token access scope', + workspace_id VARCHAR(64) NOT NULL DEFAULT 'default' COMMENT 'Token workspace boundary', + status TINYINT NOT NULL DEFAULT 0 COMMENT 'Token status, 0 means active', + creator VARCHAR(64) COMMENT 'Token creator', + gmt_create DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time', + expire_time DATETIME NULL COMMENT 'Expire time, null means long-lived', + last_used_time DATETIME NULL COMMENT 'Last used time', + revoked_time DATETIME NULL COMMENT 'Token revoked time', + revoked_by VARCHAR(64) COMMENT 'Token revoker', + UNIQUE KEY uk_hzb_auth_token_hash (token_hash), + INDEX idx_hzb_auth_token_creator (creator), + INDEX idx_hzb_auth_token_scope (token_scope), + INDEX idx_hzb_auth_token_workspace (workspace_id), + INDEX idx_hzb_auth_token_scope_workspace (token_scope, workspace_id), + INDEX idx_hzb_auth_token_status (status), + INDEX idx_hzb_auth_token_revoked_by (revoked_by) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; + +CREATE TABLE hzb_signal_saved_view ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + creator VARCHAR(255) NOT NULL COMMENT 'Saved view creator', + `signal` VARCHAR(32) NOT NULL COMMENT 'Signal type: logs, traces, or metrics', + view_key VARCHAR(128) NOT NULL COMMENT 'Stable saved view key', + label VARCHAR(255) NOT NULL COMMENT 'Saved view display label', + description VARCHAR(512) COMMENT 'Saved view description', + route VARCHAR(2048) NOT NULL COMMENT 'Explorer route snapshot', + query_snapshot TEXT COMMENT 'Query-state snapshot JSON', + payload TEXT COMMENT 'Additional saved view payload JSON', + create_time DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time', + update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Update time', + UNIQUE KEY uk_hzb_signal_saved_view_signal_key (`signal`, view_key), + INDEX idx_hzb_signal_saved_view_signal (`signal`), + INDEX idx_hzb_signal_saved_view_update (update_time) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; + +CREATE TABLE hzb_signal_dashboard_panel_draft ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + creator VARCHAR(255) NOT NULL COMMENT 'Panel draft creator', + `signal` VARCHAR(32) NOT NULL COMMENT 'Signal type: logs, traces, or metrics', + draft_key VARCHAR(128) NOT NULL COMMENT 'Stable dashboard panel draft key', + title VARCHAR(255) NOT NULL COMMENT 'Dashboard panel title', + description VARCHAR(512) COMMENT 'Dashboard panel description', + visualization VARCHAR(32) NOT NULL COMMENT 'Dashboard panel visualization type', + route VARCHAR(2048) NOT NULL COMMENT 'Explorer route snapshot', + query_snapshot TEXT COMMENT 'Query-state snapshot JSON', + payload TEXT COMMENT 'Additional dashboard panel payload JSON', + create_time DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time', + update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Update time', + UNIQUE KEY uk_hzb_signal_dashboard_panel_draft_creator_signal_key (creator, `signal`, draft_key), + INDEX idx_hzb_signal_dashboard_panel_draft_creator_signal (creator, `signal`), + INDEX idx_hzb_signal_dashboard_panel_draft_update (update_time) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; + +CREATE TABLE hzb_signal_dashboard ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + creator VARCHAR(255) NOT NULL COMMENT 'Dashboard creator', + dashboard_key VARCHAR(128) NOT NULL COMMENT 'Stable dashboard key', + title VARCHAR(255) NOT NULL COMMENT 'Dashboard title', + description VARCHAR(512) COMMENT 'Dashboard description', + tags VARCHAR(512) COMMENT 'Comma-separated dashboard tags', + layout TEXT NOT NULL COMMENT 'Dashboard layout JSON', + widgets TEXT NOT NULL COMMENT 'Dashboard widgets JSON', + variables TEXT COMMENT 'Dashboard variables JSON', + panel_map TEXT COMMENT 'Dashboard panel grouping JSON', + version VARCHAR(32) COMMENT 'Dashboard schema version', + create_time DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time', + update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Update time', + UNIQUE KEY uk_hzb_signal_dashboard_key (dashboard_key), + INDEX idx_hzb_signal_dashboard_update (update_time) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; + +ALTER TABLE hzb_auth_token + ADD COLUMN token_audience VARCHAR(32) NULL, + ADD COLUMN collector_id VARCHAR(128) NULL, + ADD COLUMN allowed_signals VARCHAR(64) NULL, + ADD INDEX idx_hzb_auth_token_collector (collector_id); + +ALTER TABLE hzb_collector ADD COLUMN runtime_config TEXT NULL; + +ALTER TABLE hzb_collector ADD COLUMN instrumentation_intake TEXT NULL; + +ALTER TABLE hzb_config ADD COLUMN config_revision VARCHAR(36) NULL; +UPDATE hzb_config SET config_revision = UUID() WHERE config_revision IS NULL; +ALTER TABLE hzb_config MODIFY COLUMN config_revision VARCHAR(36) NOT NULL; + +CREATE TABLE IF NOT EXISTS hzb_account ( + id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, + username VARCHAR(64) NOT NULL, + password_hash VARCHAR(100) NOT NULL, + roles VARCHAR(128) NOT NULL, + credential_version BIGINT NOT NULL, + disabled BOOLEAN NOT NULL, + bootstrap_slot SMALLINT, + CONSTRAINT uk_hzb_account_username UNIQUE (username), + CONSTRAINT uk_hzb_account_bootstrap UNIQUE (bootstrap_slot) +); +CREATE TABLE IF NOT EXISTS hzb_installation ( + id SMALLINT PRIMARY KEY, + installation_fingerprint VARCHAR(64) NOT NULL UNIQUE, + complete BOOLEAN NOT NULL +); diff --git a/hertzbeat-startup/src/main/resources/db/migration/mysql/V200__create_entity_foundation.sql b/hertzbeat-startup/src/main/resources/db/migration/mysql/V200__create_entity_foundation.sql index 56c51e6a95..9e639163f8 100644 --- a/hertzbeat-startup/src/main/resources/db/migration/mysql/V200__create_entity_foundation.sql +++ b/hertzbeat-startup/src/main/resources/db/migration/mysql/V200__create_entity_foundation.sql @@ -176,7 +176,7 @@ CREATE TABLE hzb_auth_token ( CREATE TABLE hzb_signal_saved_view ( id BIGINT AUTO_INCREMENT PRIMARY KEY, creator VARCHAR(255) NOT NULL COMMENT 'Saved view creator', - signal VARCHAR(32) NOT NULL COMMENT 'Signal type: logs, traces, or metrics', + `signal` VARCHAR(32) NOT NULL COMMENT 'Signal type: logs, traces, or metrics', view_key VARCHAR(128) NOT NULL COMMENT 'Stable saved view key', label VARCHAR(255) NOT NULL COMMENT 'Saved view display label', description VARCHAR(512) COMMENT 'Saved view description', @@ -185,15 +185,15 @@ CREATE TABLE hzb_signal_saved_view ( payload TEXT COMMENT 'Additional saved view payload JSON', create_time DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time', update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Update time', - UNIQUE KEY uk_hzb_signal_saved_view_signal_key (signal, view_key), - INDEX idx_hzb_signal_saved_view_signal (signal), + UNIQUE KEY uk_hzb_signal_saved_view_signal_key (`signal`, view_key), + INDEX idx_hzb_signal_saved_view_signal (`signal`), INDEX idx_hzb_signal_saved_view_update (update_time) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; CREATE TABLE hzb_signal_dashboard_panel_draft ( id BIGINT AUTO_INCREMENT PRIMARY KEY, creator VARCHAR(255) NOT NULL COMMENT 'Panel draft creator', - signal VARCHAR(32) NOT NULL COMMENT 'Signal type: logs, traces, or metrics', + `signal` VARCHAR(32) NOT NULL COMMENT 'Signal type: logs, traces, or metrics', draft_key VARCHAR(128) NOT NULL COMMENT 'Stable dashboard panel draft key', title VARCHAR(255) NOT NULL COMMENT 'Dashboard panel title', description VARCHAR(512) COMMENT 'Dashboard panel description', @@ -203,8 +203,8 @@ CREATE TABLE hzb_signal_dashboard_panel_draft ( payload TEXT COMMENT 'Additional dashboard panel payload JSON', create_time DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT 'Create time', update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Update time', - UNIQUE KEY uk_hzb_signal_dashboard_panel_draft_creator_signal_key (creator, signal, draft_key), - INDEX idx_hzb_signal_dashboard_panel_draft_creator_signal (creator, signal), + UNIQUE KEY uk_hzb_signal_dashboard_panel_draft_creator_signal_key (creator, `signal`, draft_key), + INDEX idx_hzb_signal_dashboard_panel_draft_creator_signal (creator, `signal`), INDEX idx_hzb_signal_dashboard_panel_draft_update (update_time) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; diff --git a/hertzbeat-startup/src/main/resources/db/migration/mysql/V206__normalize_signal_identifier.sql b/hertzbeat-startup/src/main/resources/db/migration/mysql/V206__normalize_signal_identifier.sql new file mode 100644 index 0000000000..e064837165 --- /dev/null +++ b/hertzbeat-startup/src/main/resources/db/migration/mysql/V206__normalize_signal_identifier.sql @@ -0,0 +1,21 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +ALTER TABLE hzb_signal_saved_view + MODIFY COLUMN `signal` VARCHAR(32) NOT NULL COMMENT 'Signal type: logs, traces, or metrics'; +ALTER TABLE hzb_signal_dashboard_panel_draft + MODIFY COLUMN `signal` VARCHAR(32) NOT NULL COMMENT 'Signal type: logs, traces, or metrics'; diff --git a/hertzbeat-startup/src/main/resources/db/migration/postgresql/B206__current_schema.sql b/hertzbeat-startup/src/main/resources/db/migration/postgresql/B206__current_schema.sql new file mode 100644 index 0000000000..248da18dc7 --- /dev/null +++ b/hertzbeat-startup/src/main/resources/db/migration/postgresql/B206__current_schema.sql @@ -0,0 +1,905 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0. +-- +-- Static V206 schema baseline for provisioning an empty PostgreSQL target. +-- Future versioned migrations start at V207 or later. + + create table hzb_ai_conversation ( + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + creator varchar(255), + modifier varchar(255), + title varchar(255), + primary key (id) + ); + + create table hzb_ai_message ( + conversation_id bigint, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + creator varchar(255), + modifier varchar(255), + role varchar(255), + content oid not null, + primary key (id) + ); + + create table hzb_alert_define ( + enable boolean not null, + period integer, + times integer, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + datasource varchar(100), + name varchar(100) not null, + expr varchar(2048), + labels varchar(2048), + template varchar(2048), + annotations varchar(4096), + creator varchar(255), + modifier varchar(255), + type varchar(255), + primary key (id) + ); + + create table hzb_alert_define_monitor_bind ( + alert_define_id bigint, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + monitor_id bigint, + primary key (id) + ); + + create table hzb_alert_group ( + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + common_labels varchar(2048), + group_key varchar(2048), + group_labels varchar(2048), + alert_fingerprints TEXT, + common_annotations TEXT, + creator varchar(255), + modifier varchar(255), + status varchar(255), + primary key (id), + constraint unique_group_key unique (group_key) + ); + + create table hzb_alert_group_converge ( + enable boolean, + gmt_create timestamp(6), + gmt_update timestamp(6), + group_interval bigint, + group_wait bigint, + id bigint generated by default as identity, + repeat_interval bigint, + name varchar(100) not null, + group_labels varchar(1024), + creator varchar(255), + modifier varchar(255), + primary key (id) + ); + + create table hzb_alert_inhibit ( + enable boolean, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + name varchar(100) not null, + equal_labels varchar(2048), + source_labels varchar(2048), + target_labels varchar(2048), + creator varchar(255), + modifier varchar(255), + primary key (id) + ); + + create table hzb_alert_silence ( + enable boolean not null, + match_all boolean not null, + times integer, + type smallint not null, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + period_end timestamp(6) with time zone, + period_start timestamp(6) with time zone, + name varchar(100) not null, + labels varchar(2048), + creator varchar(255), + days varchar(255), + modifier varchar(255), + primary key (id) + ); + + create table hzb_alert_single ( + trigger_times integer, + active_at bigint, + end_at bigint, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + start_at bigint, + fingerprint varchar(2048), + labels varchar(2048), + annotations varchar(4096), + content varchar(4096), + creator varchar(255), + modifier varchar(255), + status varchar(255), + primary key (id), + constraint unique_fingerprint unique (fingerprint) + ); + + create table hzb_bulletin ( + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + fields varchar(4096), + monitor_ids varchar(4096), + app varchar(255), + creator varchar(255), + modifier varchar(255), + name varchar(255), + primary key (id) + ); + + create table hzb_collector ( + status smallint not null check ((status>=0)), + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + creator varchar(255), + ip varchar(255) not null, + mode varchar(255), + modifier varchar(255), + name varchar(255) not null, + version varchar(255), + + primary key (id), + unique (name) + ); + + create table hzb_collector_monitor_bind ( + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + monitor_id bigint, + collector varchar(255), + creator varchar(255), + modifier varchar(255), + primary key (id) + ); + + create table hzb_config ( + gmt_create timestamp(6), + gmt_update timestamp(6), + + content varchar(8192), + creator varchar(255), + modifier varchar(255), + type varchar(255) not null, + primary key (type) + ); + + create table hzb_define ( + gmt_create timestamp(6), + gmt_update timestamp(6), + app varchar(255) not null, + creator varchar(255), + modifier varchar(255), + content oid, + primary key (app) + ); + + create table hzb_grafana_dashboard ( + enabled boolean not null, + monitor_id bigint not null, + version bigint, + folder_uid varchar(255), + slug varchar(255), + status varchar(255), + uid varchar(255), + url varchar(255), + primary key (monitor_id) + ); + + create table hzb_history ( + dou float(53), + int32 integer, + metric_type smallint, + id bigint generated by default as identity, + time bigint, + str varchar(2048), + app varchar(255), + metric_labels varchar(5000), + metric varchar(255), + metrics varchar(255), + instance varchar(255), + primary key (id) + ); + + create table hzb_metrics_favorite ( + create_time timestamp(6), + id bigint generated by default as identity, + monitor_id bigint not null, + creator varchar(255) not null, + metrics_name varchar(255) not null, + primary key (id), + unique (creator, monitor_id, metrics_name) + ); + + create table hzb_monitor ( + intervals integer, + status smallint not null check ((status<=4) and (status>=0)), + type smallint not null, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint not null, + job_id bigint, + schedule_type varchar(20), + app varchar(100), + cron_expression varchar(100), + instance varchar(100), + name varchar(100), + scrape varchar(100), + annotations varchar(4096), + labels varchar(4096), + creator varchar(255), + description varchar(255), + modifier varchar(255), + primary key (id) + ); + + create table hzb_monitor_bind ( + biz_id bigint, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + monitor_id bigint, + creator varchar(255), + key_str varchar(255), + modifier varchar(255), + primary key (id) + ); + + create table hzb_notice_receiver ( + agent_id integer, + lark_receive_type smallint, + type smallint not null check ((type>=0)), + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + smn_ak varchar(22), + smn_project_id varchar(32), + smn_region varchar(32), + smn_sk varchar(42), + email varchar(100), + name varchar(100) not null, + phone varchar(100), + access_token varchar(300), + discord_bot_token varchar(300), + discord_channel_id varchar(300), + gotify_token varchar(300), + hook_auth_token varchar(300), + hook_auth_type varchar(300), + server_chan_token varchar(300), + slack_web_hook_url varchar(300), + smn_topic_urn varchar(300), + wechat_id varchar(300), + hook_url varchar(1000), + app_id varchar(255), + app_secret varchar(255), + chat_id varchar(255), + corp_id varchar(255), + creator varchar(255), + modifier varchar(255), + party_id varchar(255), + tag_id varchar(255), + tg_bot_token varchar(255), + tg_message_thread_id varchar(255), + tg_user_id varchar(255), + user_id varchar(255), + primary key (id) + ); + + create table hzb_notice_rule ( + enable boolean not null, + filter_all boolean not null, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + period_end timestamp(6) with time zone, + period_start timestamp(6) with time zone, + template_id bigint, + name varchar(100) not null, + template_name varchar(100), + labels varchar(2048), + creator varchar(255), + days varchar(255), + modifier varchar(255), + receiver_id varchar(255) not null, + receiver_name varchar(255), + primary key (id) + ); + + create table hzb_notice_template ( + preset boolean default false, + type smallint not null check ((type>=0)), + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + name varchar(100) not null, + creator varchar(255), + modifier varchar(255), + content oid not null, + primary key (id) + ); + + create table hzb_param ( + type smallint not null check ((type>=0)), + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + monitor_id bigint, + field varchar(100) not null, + param_value varchar(8126), + primary key (id), + constraint uk_hzb_param_monitor_field unique (monitor_id, field) + ); + + create table hzb_param_define ( + hide boolean not null, + param_limit smallint, + required boolean not null, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + name varchar(2048), + param_options varchar(2048), + app varchar(255), + creator varchar(255), + default_value varchar(255), + depend varchar(255), + field varchar(255), + key_alias varchar(255), + modifier varchar(255), + param_range varchar(255), + placeholder varchar(255), + type varchar(255), + value_alias varchar(255), + primary key (id) + ); + + create table hzb_plugin_item ( + id bigint generated by default as identity, + metadata_id bigint, + class_identifier varchar(255), + type varchar(255) check ((type in ('POST_ALERT','POST_COLLECT'))), + primary key (id) + ); + + create table hzb_plugin_metadata ( + enable_status boolean, + param_count integer, + gmt_create timestamp(6), + id bigint generated by default as identity, + creator varchar(255), + jar_file_path varchar(255), + name varchar(255) not null, + primary key (id) + ); + + create table hzb_plugin_param ( + type smallint not null check ((type>=0)), + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + plugin_metadata_id bigint not null, + field varchar(100) not null, + param_value varchar(8126), + primary key (id), + constraint uk_hzb_plugin_param_metadata_field unique (plugin_metadata_id, field) + ); + + create table hzb_push_metrics ( + id bigint generated by default as identity, + monitor_id bigint, + time bigint, + metrics varchar(255), + primary key (id) + ); + + create table hzb_sop_schedule ( + id bigserial primary key, + conversation_id bigint not null, + sop_name varchar(64) not null, + sop_params varchar(1024), + cron_expression varchar(64) not null, + enabled boolean default true, + last_run_time timestamp, + next_run_time timestamp, + creator varchar(64), + modifier varchar(64), + gmt_create timestamp default current_timestamp, + gmt_update timestamp default current_timestamp + ); + + comment on table hzb_sop_schedule is 'Scheduled SOP execution configurations'; + comment on column hzb_sop_schedule.conversation_id is 'Conversation ID to push results to'; + comment on column hzb_sop_schedule.sop_name is 'Name of the SOP skill to execute'; + comment on column hzb_sop_schedule.sop_params is 'SOP execution parameters in JSON format'; + comment on column hzb_sop_schedule.cron_expression is 'Cron expression for scheduling'; + comment on column hzb_sop_schedule.enabled is 'Whether the schedule is enabled'; + comment on column hzb_sop_schedule.last_run_time is 'Last execution time'; + comment on column hzb_sop_schedule.next_run_time is 'Next scheduled execution time'; + + create table hzb_status_page_component ( + config_state smallint not null, + method smallint not null, + state smallint not null, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + org_id bigint, + labels varchar(4096), + creator varchar(255), + description varchar(255), + modifier varchar(255), + name varchar(255) not null, + primary key (id) + ); + + create table hzb_status_page_history ( + abnormal integer, + normal integer, + state smallint not null, + unknowing integer, + uptime float(53), + component_id bigint, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + timestamp bigint, + creator varchar(255), + modifier varchar(255), + primary key (id) + ); + + create table hzb_status_page_incident ( + state smallint not null, + end_time bigint, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + org_id bigint, + start_time bigint, + creator varchar(255), + modifier varchar(255), + name varchar(255) not null, + primary key (id) + ); + + create table hzb_status_page_incident_component_bind ( + component_id bigint, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + incident_id bigint, + primary key (id) + ); + + create table hzb_status_page_incident_content ( + state smallint not null, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + incident_id bigint, + timestamp bigint, + creator varchar(255), + message TEXT not null, + modifier varchar(255), + primary key (id) + ); + + create table hzb_status_page_org ( + state smallint not null, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + color varchar(255), + creator varchar(255), + description varchar(255) not null, + feedback varchar(255), + home varchar(255) not null, + logo varchar(255) not null, + modifier varchar(255), + name varchar(255) not null, + primary key (id) + ); + + create table hzb_tag ( + type smallint check ((type<=3) and (type>=0)), + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + tag_value varchar(2048), + creator varchar(255), + description varchar(255), + modifier varchar(255), + name varchar(255) not null, + primary key (id) + ); + + create index idx_message_conversation_id + on hzb_ai_message (conversation_id); + + create index idx_alert_define_id + on hzb_alert_define_monitor_bind (alert_define_id); + + create index idx_monitor_id + on hzb_alert_define_monitor_bind (monitor_id); + + create index idx_name + on hzb_alert_group_converge (name); + + create index idx_collector_monitor_collector + on hzb_collector_monitor_bind (collector); + + create index idx_collector_monitor_monitor_id + on hzb_collector_monitor_bind (monitor_id); + + + + + + create index idx_hzb_history_instance + on hzb_history (instance); + + create index idx_hzb_history_app + on hzb_history (app); + + create index idx_hzb_history_metrics + on hzb_history (metrics); + + create index idx_hzb_history_metric + on hzb_history (metric); + + create index idx_hzb_monitor_app + on hzb_monitor (app); + + create index idx_hzb_monitor_instance + on hzb_monitor (instance); + + create index idx_hzb_monitor_name + on hzb_monitor (name); + + create index index_monitor_bind + on hzb_monitor_bind (biz_id); + + create index index_monitor_bin + on hzb_monitor_bind (monitor_id); + + create index idx_hzb_param_monitor_id + on hzb_param (monitor_id); + + create index idx_hzb_plugin_param_plugin_metadata_id + on hzb_plugin_param (plugin_metadata_id); + + create index idx_push_metrics_monitor_id + on hzb_push_metrics (monitor_id); + + create index idx_push_metrics_time + on hzb_push_metrics (time); + + create index idx_schedule_conversation_id + on hzb_sop_schedule (conversation_id); + + create index idx_schedule_enabled_next + on hzb_sop_schedule (enabled, next_run_time); + + create index index_incident_component + on hzb_status_page_incident_component_bind (incident_id); + + create index idx_incident_component_component_id + on hzb_status_page_incident_component_bind (component_id); + + alter table if exists hzb_ai_message + add constraint fk_hzb_ai_message_conversation + foreign key (conversation_id) + references hzb_ai_conversation; + + alter table if exists hzb_plugin_item + add constraint fk_hzb_plugin_item_metadata + foreign key (metadata_id) + references hzb_plugin_metadata; + + alter table if exists hzb_status_page_incident_content + add constraint fk_hzb_incident_content_incident + foreign key (incident_id) + references hzb_status_page_incident; + +CREATE TABLE hzb_entity ( + id BIGINT PRIMARY KEY, + entity_type VARCHAR(32) NOT NULL, + name VARCHAR(128) NOT NULL, + display_name VARCHAR(128), + sub_type VARCHAR(128), + namespace VARCHAR(128), + environment VARCHAR(128), + status VARCHAR(32) NOT NULL, + criticality VARCHAR(32), + owner VARCHAR(128), + additional_owners TEXT, + runbook VARCHAR(512), + lifecycle VARCHAR(64), + tier VARCHAR(64), + system_name VARCHAR(128), + component_of TEXT, + components TEXT, + implemented_by TEXT, + api_interface TEXT, + inherit_from VARCHAR(255), + languages TEXT, + links TEXT, + contacts TEXT, + integrations TEXT, + extensions TEXT, + hertzbeat TEXT, + source VARCHAR(32) NOT NULL, + description VARCHAR(512), + labels VARCHAR(4096), + tags TEXT, + workspace_id VARCHAR(64) NOT NULL DEFAULT 'default', + creator VARCHAR(64), + modifier VARCHAR(64), + gmt_create TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + gmt_update TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX idx_hzb_entity_type ON hzb_entity(entity_type); +CREATE INDEX idx_hzb_entity_status ON hzb_entity(status); +CREATE INDEX idx_hzb_entity_name ON hzb_entity(name); +CREATE INDEX idx_hzb_entity_owner ON hzb_entity(owner); +CREATE INDEX idx_hzb_entity_workspace ON hzb_entity(workspace_id); + +CREATE TABLE hzb_entity_identity ( + id BIGSERIAL PRIMARY KEY, + entity_id BIGINT NOT NULL, + identity_type VARCHAR(32) NOT NULL, + identity_key VARCHAR(128) NOT NULL, + identity_value VARCHAR(512) NOT NULL, + normalized_value VARCHAR(512) NOT NULL, + priority INT NOT NULL DEFAULT 40, + primary_identity BOOLEAN NOT NULL DEFAULT FALSE, + creator VARCHAR(64), + modifier VARCHAR(64), + gmt_create TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + gmt_update TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE UNIQUE INDEX uk_hzb_entity_identity ON hzb_entity_identity(entity_id, identity_key, normalized_value); +CREATE INDEX idx_hzb_entity_identity_lookup ON hzb_entity_identity(identity_key, normalized_value); +CREATE INDEX idx_hzb_entity_identity_entity ON hzb_entity_identity(entity_id); + +CREATE TABLE hzb_entity_monitor_bind ( + id BIGSERIAL PRIMARY KEY, + entity_id BIGINT NOT NULL, + monitor_id BIGINT NOT NULL, + bind_type VARCHAR(32) NOT NULL, + bind_source VARCHAR(64) NOT NULL, + status VARCHAR(16) NOT NULL, + score INT NOT NULL DEFAULT 100, + match_context TEXT, + creator VARCHAR(64), + modifier VARCHAR(64), + gmt_create TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + gmt_update TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE UNIQUE INDEX uk_hzb_entity_monitor_bind ON hzb_entity_monitor_bind(entity_id, monitor_id); +CREATE INDEX idx_hzb_entity_monitor_bind_entity ON hzb_entity_monitor_bind(entity_id); +CREATE INDEX idx_hzb_entity_monitor_bind_monitor ON hzb_entity_monitor_bind(monitor_id); + +CREATE TABLE hzb_entity_relation ( + id BIGSERIAL PRIMARY KEY, + source_entity_id BIGINT NOT NULL, + target_entity_id BIGINT, + target_ref VARCHAR(255), + relation_type VARCHAR(32) NOT NULL, + relation_source VARCHAR(32) NOT NULL, + status VARCHAR(16) NOT NULL, + score INT NOT NULL DEFAULT 100, + description VARCHAR(255), + attributes TEXT, + creator VARCHAR(64), + modifier VARCHAR(64), + gmt_create TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + gmt_update TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE UNIQUE INDEX uk_hzb_entity_relation ON hzb_entity_relation(source_entity_id, target_entity_id, relation_type); +CREATE INDEX idx_hzb_entity_relation_source ON hzb_entity_relation(source_entity_id); +CREATE INDEX idx_hzb_entity_relation_target ON hzb_entity_relation(target_entity_id); +CREATE INDEX idx_hzb_entity_relation_target_ref ON hzb_entity_relation(target_ref); + +CREATE TABLE hzb_entity_definition_activity ( + id BIGSERIAL PRIMARY KEY, + entity_id BIGINT NOT NULL, + workspace_id VARCHAR(64) NOT NULL DEFAULT 'default', + activity_type VARCHAR(32) NOT NULL, + format VARCHAR(16) NOT NULL, + status VARCHAR(16) NOT NULL, + summary VARCHAR(128) NOT NULL, + detail VARCHAR(255), + creator VARCHAR(64), + gmt_create TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX idx_hzb_entity_definition_activity_entity + ON hzb_entity_definition_activity(entity_id); + +CREATE INDEX idx_hzb_entity_definition_activity_workspace_time + ON hzb_entity_definition_activity(workspace_id, gmt_create); + +CREATE INDEX idx_hzb_entity_definition_activity_time + ON hzb_entity_definition_activity(gmt_create); + +CREATE TABLE hzb_entity_governance_state ( + id BIGSERIAL PRIMARY KEY, + state_scope VARCHAR(32) NOT NULL, + state_kind VARCHAR(32) NOT NULL, + workspace_id VARCHAR(64) NOT NULL DEFAULT 'default', + state_key VARCHAR(128) NOT NULL, + state_name VARCHAR(128), + status VARCHAR(32), + content TEXT, + creator VARCHAR(64), + gmt_create TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + gmt_update TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE UNIQUE INDEX uk_hzb_entity_governance_state_scope_kind_workspace_key + ON hzb_entity_governance_state(state_scope, state_kind, workspace_id, state_key); + +CREATE INDEX idx_hzb_entity_governance_state_scope_kind + ON hzb_entity_governance_state(state_scope, state_kind); + +CREATE INDEX idx_hzb_entity_governance_state_scope_kind_workspace + ON hzb_entity_governance_state(state_scope, state_kind, workspace_id); + +CREATE INDEX idx_hzb_entity_governance_state_update + ON hzb_entity_governance_state(gmt_update); + +CREATE INDEX idx_hzb_entity_governance_state_creator + ON hzb_entity_governance_state(creator); + +CREATE TABLE hzb_auth_token ( + id BIGSERIAL PRIMARY KEY, + name VARCHAR(255), + token_hash VARCHAR(128) NOT NULL, + token_mask VARCHAR(64), + token_scope VARCHAR(32) NOT NULL DEFAULT 'api-admin', + workspace_id VARCHAR(64) NOT NULL DEFAULT 'default', + status SMALLINT NOT NULL DEFAULT 0, + creator VARCHAR(64), + gmt_create TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + expire_time TIMESTAMP NULL, + last_used_time TIMESTAMP NULL, + revoked_time TIMESTAMP NULL, + revoked_by VARCHAR(64) +); + +CREATE UNIQUE INDEX uk_hzb_auth_token_hash ON hzb_auth_token(token_hash); +CREATE INDEX idx_hzb_auth_token_creator ON hzb_auth_token(creator); +CREATE INDEX idx_hzb_auth_token_scope ON hzb_auth_token(token_scope); +CREATE INDEX idx_hzb_auth_token_workspace ON hzb_auth_token(workspace_id); +CREATE INDEX idx_hzb_auth_token_scope_workspace ON hzb_auth_token(token_scope, workspace_id); +CREATE INDEX idx_hzb_auth_token_status ON hzb_auth_token(status); +CREATE INDEX idx_hzb_auth_token_revoked_by ON hzb_auth_token(revoked_by); + +CREATE TABLE hzb_signal_saved_view ( + id BIGSERIAL PRIMARY KEY, + creator VARCHAR(255) NOT NULL, + signal VARCHAR(32) NOT NULL, + view_key VARCHAR(128) NOT NULL, + label VARCHAR(255) NOT NULL, + description VARCHAR(512), + route VARCHAR(2048) NOT NULL, + query_snapshot TEXT, + payload TEXT, + create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE UNIQUE INDEX uk_hzb_signal_saved_view_signal_key + ON hzb_signal_saved_view(signal, view_key); + +CREATE INDEX idx_hzb_signal_saved_view_signal + ON hzb_signal_saved_view(signal); + +CREATE INDEX idx_hzb_signal_saved_view_update + ON hzb_signal_saved_view(update_time); + +CREATE TABLE hzb_signal_dashboard_panel_draft ( + id BIGSERIAL PRIMARY KEY, + creator VARCHAR(255) NOT NULL, + signal VARCHAR(32) NOT NULL, + draft_key VARCHAR(128) NOT NULL, + title VARCHAR(255) NOT NULL, + description VARCHAR(512), + visualization VARCHAR(32) NOT NULL, + route VARCHAR(2048) NOT NULL, + query_snapshot TEXT, + payload TEXT, + create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE UNIQUE INDEX uk_hzb_signal_dashboard_panel_draft_creator_signal_key + ON hzb_signal_dashboard_panel_draft(creator, signal, draft_key); + +CREATE INDEX idx_hzb_signal_dashboard_panel_draft_creator_signal + ON hzb_signal_dashboard_panel_draft(creator, signal); + +CREATE INDEX idx_hzb_signal_dashboard_panel_draft_update + ON hzb_signal_dashboard_panel_draft(update_time); + +CREATE TABLE hzb_signal_dashboard ( + id BIGSERIAL PRIMARY KEY, + creator VARCHAR(255) NOT NULL, + dashboard_key VARCHAR(128) NOT NULL, + title VARCHAR(255) NOT NULL, + description VARCHAR(512), + tags VARCHAR(512), + layout TEXT NOT NULL, + widgets TEXT NOT NULL, + variables TEXT, + panel_map TEXT, + version VARCHAR(32), + create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE UNIQUE INDEX uk_hzb_signal_dashboard_key + ON hzb_signal_dashboard(dashboard_key); + +CREATE INDEX idx_hzb_signal_dashboard_update + ON hzb_signal_dashboard(update_time); + +ALTER TABLE hzb_auth_token ADD COLUMN token_audience VARCHAR(32); +ALTER TABLE hzb_auth_token ADD COLUMN collector_id VARCHAR(128); +ALTER TABLE hzb_auth_token ADD COLUMN allowed_signals VARCHAR(64); +CREATE INDEX idx_hzb_auth_token_collector ON hzb_auth_token(collector_id); + +ALTER TABLE hzb_collector ADD COLUMN runtime_config TEXT; + +ALTER TABLE hzb_collector ADD COLUMN instrumentation_intake TEXT; + +ALTER TABLE hzb_config ADD COLUMN config_revision VARCHAR(36); +UPDATE hzb_config SET config_revision = gen_random_uuid()::text WHERE config_revision IS NULL; +ALTER TABLE hzb_config ALTER COLUMN config_revision SET NOT NULL; + +CREATE TABLE IF NOT EXISTS hzb_account ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + username VARCHAR(64) NOT NULL, + password_hash VARCHAR(100) NOT NULL, + roles VARCHAR(128) NOT NULL, + credential_version BIGINT NOT NULL, + disabled BOOLEAN NOT NULL, + bootstrap_slot SMALLINT, + CONSTRAINT uk_hzb_account_username UNIQUE (username), + CONSTRAINT uk_hzb_account_bootstrap UNIQUE (bootstrap_slot) +); +CREATE TABLE IF NOT EXISTS hzb_installation ( + id SMALLINT PRIMARY KEY, + installation_fingerprint VARCHAR(64) NOT NULL UNIQUE, + complete BOOLEAN NOT NULL +); diff --git a/hertzbeat-startup/src/main/resources/db/migration/postgresql/V206__normalize_sop_schedule_enabled.sql b/hertzbeat-startup/src/main/resources/db/migration/postgresql/V206__normalize_sop_schedule_enabled.sql new file mode 100644 index 0000000000..be82d77274 --- /dev/null +++ b/hertzbeat-startup/src/main/resources/db/migration/postgresql/V206__normalize_sop_schedule_enabled.sql @@ -0,0 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +ALTER TABLE hzb_sop_schedule ALTER COLUMN enabled DROP DEFAULT; +ALTER TABLE hzb_sop_schedule ALTER COLUMN enabled TYPE BOOLEAN USING enabled <> 0; +ALTER TABLE hzb_sop_schedule ALTER COLUMN enabled SET DEFAULT TRUE; diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HistoricalMetadataSchema.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HistoricalMetadataSchema.java new file mode 100644 index 0000000000..a2222e7cef --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/HistoricalMetadataSchema.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import org.flywaydb.core.Flyway; + +/** Rebuilds the current schema from an independent V159 fixture and the committed migrations. */ +final class HistoricalMetadataSchema { + + private static final String FIXTURE = "V159__schema.sql"; + + private HistoricalMetadataSchema() { + } + + static void rebuild(String jdbcUrl, String username, String password, String vendor) + throws SQLException, IOException { + Flyway.configure() + .dataSource(jdbcUrl, username, password) + .locations("classpath:db/migration/" + vendor) + .cleanDisabled(false) + .load() + .clean(); + try (Connection connection = DriverManager.getConnection(jdbcUrl, username, password)) { + executeFixture(connection, resource(vendor)); + } + Flyway flyway = Flyway.configure() + .dataSource(jdbcUrl, username, password) + .locations("classpath:db/migration/" + vendor) + .baselineVersion("159") + .baselineOnMigrate(true) + .cleanDisabled(true) + .target("206") + .validateMigrationNaming(true) + .load(); + flyway.migrate(); + flyway.validate(); + } + + private static void executeFixture(Connection connection, String fixture) throws SQLException { + String executable = fixture.replaceAll("(?m)^--.*$", ""); + try (Statement statement = connection.createStatement()) { + for (String sql : executable.split(";")) { + if (!sql.isBlank()) { + statement.execute(sql); + } + } + } + } + + private static String resource(String vendor) throws IOException { + String path = "/db/historical/" + vendor + '/' + FIXTURE; + try (InputStream input = HistoricalMetadataSchema.class.getResourceAsStream(path)) { + if (input == null) { + throw new IOException("Historical schema fixture is missing: " + path); + } + return new String(input.readAllBytes(), StandardCharsets.UTF_8); + } + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataSchemaSnapshot.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataSchemaSnapshot.java new file mode 100644 index 0000000000..ac217880f4 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataSchemaSnapshot.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** Normalized JDBC metadata used to compare two schemas on the same database vendor. */ +record MetadataSchemaSnapshot( + Set columns, + Set primaryKeys, + Set indexes, + Set foreignKeys) { + + static MetadataSchemaSnapshot capture(Connection connection) throws SQLException { + DatabaseMetaData metadata = connection.getMetaData(); + Set tables = tables(connection, metadata); + Set columns = new HashSet<>(); + Set primaryKeys = new HashSet<>(); + Set indexes = new HashSet<>(); + Set foreignKeys = new HashSet<>(); + for (String table : tables) { + readColumns(connection, metadata, table, columns); + readPrimaryKeys(connection, metadata, table, primaryKeys); + readIndexes(connection, metadata, table, indexes); + readForeignKeys(connection, metadata, table, foreignKeys); + } + return new MetadataSchemaSnapshot(columns, primaryKeys, indexes, foreignKeys); + } + + private static Set tables(Connection connection, DatabaseMetaData metadata) throws SQLException { + Set tables = new HashSet<>(); + try (ResultSet result = metadata.getTables(connection.getCatalog(), null, "hzb_%", new String[]{"TABLE"})) { + while (result.next()) { + tables.add(normalize(result.getString("TABLE_NAME"))); + } + } + return tables; + } + + private static void readColumns( + Connection connection, DatabaseMetaData metadata, String table, Set columns) throws SQLException { + try (ResultSet result = metadata.getColumns(connection.getCatalog(), null, table, null)) { + while (result.next()) { + columns.add(new Column( + table, + normalize(result.getString("COLUMN_NAME")), + result.getInt("DATA_TYPE"), + normalize(result.getString("TYPE_NAME")), + result.getInt("COLUMN_SIZE"), + result.getInt("DECIMAL_DIGITS"), + result.getInt("NULLABLE"), + normalize(result.getString("COLUMN_DEF")), + normalize(result.getString("REMARKS")), + result.getInt("ORDINAL_POSITION"))); + } + } + } + + private static void readPrimaryKeys( + Connection connection, DatabaseMetaData metadata, String table, Set primaryKeys) + throws SQLException { + try (ResultSet result = metadata.getPrimaryKeys(connection.getCatalog(), null, table)) { + while (result.next()) { + primaryKeys.add(new PrimaryKey( + table, + normalize(result.getString("PK_NAME")), + result.getShort("KEY_SEQ"), + normalize(result.getString("COLUMN_NAME")))); + } + } + } + + private static void readIndexes( + Connection connection, DatabaseMetaData metadata, String table, Set indexes) throws SQLException { + Map collected = new HashMap<>(); + try (ResultSet result = metadata.getIndexInfo(connection.getCatalog(), null, table, false, false)) { + while (result.next()) { + String name = normalize(result.getString("INDEX_NAME")); + String column = normalize(result.getString("COLUMN_NAME")); + if (name == null || column == null || result.getShort("TYPE") == DatabaseMetaData.tableIndexStatistic) { + continue; + } + String key = name + ':' + result.getBoolean("NON_UNIQUE"); + collected.computeIfAbsent(key, + ignored -> new IndexAccumulator(table, name, !resultBoolean(result, "NON_UNIQUE"))) + .add(resultShort(result, "ORDINAL_POSITION"), column, normalize(resultString(result, "ASC_OR_DESC"))); + } + } + collected.values().stream().map(IndexAccumulator::build).forEach(indexes::add); + } + + private static void readForeignKeys( + Connection connection, DatabaseMetaData metadata, String table, Set foreignKeys) + throws SQLException { + try (ResultSet result = metadata.getImportedKeys(connection.getCatalog(), null, table)) { + while (result.next()) { + foreignKeys.add(new ForeignKey( + table, + normalize(result.getString("FK_NAME")), + result.getShort("KEY_SEQ"), + normalize(result.getString("FKCOLUMN_NAME")), + normalize(result.getString("PKTABLE_NAME")), + normalize(result.getString("PKCOLUMN_NAME")), + result.getShort("UPDATE_RULE"), + result.getShort("DELETE_RULE"), + result.getShort("DEFERRABILITY"))); + } + } + } + + private static boolean resultBoolean(ResultSet result, String column) { + try { + return result.getBoolean(column); + } catch (SQLException exception) { + throw new IllegalStateException("JDBC metadata result is incomplete", exception); + } + } + + private static short resultShort(ResultSet result, String column) { + try { + return result.getShort(column); + } catch (SQLException exception) { + throw new IllegalStateException("JDBC metadata result is incomplete", exception); + } + } + + private static String resultString(ResultSet result, String column) { + try { + return result.getString(column); + } catch (SQLException exception) { + throw new IllegalStateException("JDBC metadata result is incomplete", exception); + } + } + + private static String normalize(String value) { + return value == null ? null : value.toLowerCase(Locale.ROOT).replaceAll("\\s+", " ").trim(); + } + + record Column( + String table, + String name, + int jdbcType, + String typeName, + int size, + int scale, + int nullable, + String defaultValue, + String remarks, + int position) { + } + + record PrimaryKey(String table, String name, short position, String column) { + } + + record Index(String table, String name, boolean unique, List columns) { + } + + record IndexColumn(short position, String name, String order) { + } + + record ForeignKey( + String table, + String name, + short position, + String column, + String referencedTable, + String referencedColumn, + short updateRule, + short deleteRule, + short deferrability) { + } + + private static final class IndexAccumulator { + private final String table; + private final String name; + private final boolean unique; + private final List columns = new ArrayList<>(); + + private IndexAccumulator(String table, String name, boolean unique) { + this.table = table; + this.name = name; + this.unique = unique; + } + + void add(short position, String column, String order) { + columns.add(new IndexColumn(position, column, order)); + } + + Index build() { + columns.sort(java.util.Comparator.comparingInt(IndexColumn::position)); + return new Index(table, name, unique, List.copyOf(columns)); + } + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataValidationMySqlDialect.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataValidationMySqlDialect.java new file mode 100644 index 0000000000..492eb5d327 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataValidationMySqlDialect.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Types; +import org.hibernate.dialect.MySQLDialect; + +/** Keeps schema validation compatible with boolean encodings used by the committed MySQL migrations. */ +public final class MetadataValidationMySqlDialect extends MySQLDialect { + + @Override + public boolean equivalentTypes(int firstTypeCode, int secondTypeCode) { + if (isBooleanType(firstTypeCode) && isBooleanType(secondTypeCode)) { + return true; + } + return super.equivalentTypes(firstTypeCode, secondTypeCode); + } + + private static boolean isBooleanType(int typeCode) { + return typeCode == Types.BIT || typeCode == Types.BOOLEAN || typeCode == Types.TINYINT; + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaBaselineResourceTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaBaselineResourceTest.java new file mode 100644 index 0000000000..0b8ada5a5b --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaBaselineResourceTest.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.Locale; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Static schema contract for provisioning an empty external metadata database. */ +class TargetSchemaBaselineResourceTest { + + private static final Pattern CREATE_TABLE = Pattern.compile( + "(?im)^\\s*create\\s+table\\s+(?:if\\s+not\\s+exists\\s+)?([a-z][a-z0-9_]*)\\s*\\("); + private static final Set MAPPED_TABLES = Set.of( + "hzb_account", + "hzb_ai_conversation", + "hzb_ai_message", + "hzb_alert_define", + "hzb_alert_define_monitor_bind", + "hzb_alert_group", + "hzb_alert_group_converge", + "hzb_alert_inhibit", + "hzb_alert_silence", + "hzb_alert_single", + "hzb_auth_token", + "hzb_bulletin", + "hzb_collector", + "hzb_collector_monitor_bind", + "hzb_config", + "hzb_define", + "hzb_entity", + "hzb_entity_definition_activity", + "hzb_entity_governance_state", + "hzb_entity_identity", + "hzb_entity_monitor_bind", + "hzb_entity_relation", + "hzb_grafana_dashboard", + "hzb_history", + "hzb_installation", + "hzb_metrics_favorite", + "hzb_monitor", + "hzb_monitor_bind", + "hzb_notice_receiver", + "hzb_notice_rule", + "hzb_notice_template", + "hzb_param", + "hzb_param_define", + "hzb_plugin_item", + "hzb_plugin_metadata", + "hzb_plugin_param", + "hzb_push_metrics", + "hzb_signal_dashboard", + "hzb_signal_dashboard_panel_draft", + "hzb_signal_saved_view", + "hzb_sop_schedule", + "hzb_status_page_component", + "hzb_status_page_history", + "hzb_status_page_incident", + "hzb_status_page_incident_component_bind", + "hzb_status_page_incident_content", + "hzb_status_page_org", + "hzb_tag"); + + @ParameterizedTest + @ValueSource(strings = {"mysql", "postgresql"}) + void currentBaselineDeclaresEveryMappedTable(String vendor) throws IOException { + String resource = "db/migration/" + vendor + "/B206__current_schema.sql"; + try (InputStream input = getClass().getClassLoader().getResourceAsStream(resource)) { + assertThat(input).as(resource).isNotNull(); + assertThat(createdTables(new String(input.readAllBytes(), StandardCharsets.UTF_8))) + .containsExactlyInAnyOrderElementsOf(MAPPED_TABLES); + } + MetadataDatabaseKind kind = vendor.equals("mysql") + ? MetadataDatabaseKind.MYSQL : MetadataDatabaseKind.POSTGRESQL; + assertThat(TargetSchemaBaseline.load(kind).expectedTables()) + .containsExactlyInAnyOrderElementsOf(MAPPED_TABLES); + } + + @ParameterizedTest + @ValueSource(strings = {"mysql", "postgresql"}) + void historicalFixtureDeclaresImmutableV159Provenance(String vendor) throws IOException { + String resource = "db/historical/" + vendor + "/V159__schema.sql"; + try (InputStream input = getClass().getClassLoader().getResourceAsStream(resource)) { + assertThat(input).as(resource).isNotNull(); + String sql = new String(input.readAllBytes(), StandardCharsets.UTF_8); + assertThat(sql) + .contains("Immutable V159 schema fixture for migration-chain tests.") + .contains("Do not derive this fixture from the current baseline or later migrations.") + .doesNotContain("Static V205 schema baseline", "Future versioned migrations start at V206"); + } + } + + private static Set createdTables(String sql) { + Matcher matcher = CREATE_TABLE.matcher(sql.toLowerCase(Locale.ROOT)); + Set tables = new HashSet<>(); + while (matcher.find()) { + tables.add(matcher.group(1)); + } + return Set.copyOf(tables); + } + + static Set mappedTables() { + return MAPPED_TABLES; + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisionerDatabaseTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisionerDatabaseTest.java new file mode 100644 index 0000000000..9e7af00c86 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisionerDatabaseTest.java @@ -0,0 +1,481 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import jakarta.persistence.Entity; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.flywaydb.core.Flyway; +import org.assertj.core.api.SoftAssertions; +import org.hibernate.SessionFactory; +import org.hibernate.boot.MetadataSources; +import org.hibernate.boot.registry.StandardServiceRegistry; +import org.hibernate.boot.registry.StandardServiceRegistryBuilder; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.ClassPathScanningCandidateComponentProvider; +import org.springframework.core.type.filter.AnnotationTypeFilter; +import org.testcontainers.mysql.MySQLContainer; +import org.testcontainers.postgresql.PostgreSQLContainer; + +/** Real-database proof for current-version target schema provisioning. */ +@EnabledIfSystemProperty(named = "hertzbeat.test.database-containers", matches = "true") +class TargetSchemaProvisionerDatabaseTest { + + private static final String DATABASE = "hertzbeat"; + private static final String USERNAME = "hertzbeat"; + private static final String PASSWORD = "test-only-password"; + + @Test + void provisionsAndValidatesFreshMysqlSchema() throws Exception { + try (MySQLContainer database = new MySQLContainer("mysql:8.4") + .withDatabaseName(DATABASE) + .withUsername(USERNAME) + .withPassword(PASSWORD) + .withCommand("--lower-case-table-names=1")) { + database.start(); + assertRejectsFalseEmptyStates(database.getJdbcUrl(), MetadataDatabaseKind.MYSQL); + verify(database.getJdbcUrl(), MetadataDatabaseKind.MYSQL, + MetadataValidationMySqlDialect.class.getName()); + } + } + + @Test + void provisionsAndValidatesFreshPostgresqlSchema() throws Exception { + try (PostgreSQLContainer database = new PostgreSQLContainer("postgres:17.6") + .withDatabaseName(DATABASE) + .withUsername(USERNAME) + .withPassword(PASSWORD)) { + database.start(); + assertRejectsFalseEmptyStates(database.getJdbcUrl(), MetadataDatabaseKind.POSTGRESQL); + verify(database.getJdbcUrl(), MetadataDatabaseKind.POSTGRESQL, + "org.hibernate.dialect.PostgreSQLDialect"); + } + } + + private static void assertRejectsFalseEmptyStates(String jdbcUrl, MetadataDatabaseKind kind) throws Exception { + TargetSchemaProvisioner provisioner = new FlywayTargetSchemaProvisioner(); + MetadataDatabaseConfiguration target = + new MetadataDatabaseConfiguration(kind, jdbcUrl, USERNAME, PASSWORD); + SoftAssertions softly = new SoftAssertions(); + + try (Connection connection = DriverManager.getConnection(jdbcUrl, USERNAME, PASSWORD)) { + TargetSchemaBaseline baseline = TargetSchemaBaseline.load(kind); + new FlywaySchemaHistory(kind).record(connection, baseline, USERNAME, 0); + } + softly.assertThatThrownBy(() -> provisioner.provision(target)) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, exception -> + softly.assertThat(exception.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.PRECONDITION)); + try (Connection connection = DriverManager.getConnection(jdbcUrl, USERNAME, PASSWORD); + Statement statement = connection.createStatement()) { + statement.execute("CREATE TABLE hzb_account (id INTEGER NOT NULL PRIMARY KEY)"); + } + softly.assertThatThrownBy(() -> provisioner.provision(target)) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, exception -> + softly.assertThat(exception.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.PRECONDITION)); + try (Connection connection = DriverManager.getConnection(jdbcUrl, USERNAME, PASSWORD); + Statement statement = connection.createStatement()) { + softly.assertThat(metadataTables(connection)).containsExactly("hzb_account"); + statement.execute("DROP TABLE hzb_account"); + statement.execute("DROP TABLE flyway_schema_history"); + statement.execute("DROP TABLE flyway_schema_contract"); + statement.execute("CREATE TABLE unrelated_table (id INTEGER NOT NULL PRIMARY KEY)"); + } + softly.assertThatThrownBy(() -> provisioner.provision(target)) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, exception -> + softly.assertThat(exception.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.PRECONDITION)); + try (Connection connection = DriverManager.getConnection(jdbcUrl, USERNAME, PASSWORD); + Statement statement = connection.createStatement()) { + softly.assertThat(metadataTables(connection)).isEmpty(); + statement.execute("DROP TABLE unrelated_table"); + statement.execute("CREATE VIEW hzb_status_page_org AS SELECT 1 AS id"); + } + softly.assertThatThrownBy(() -> provisioner.provision(target)) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, exception -> + softly.assertThat(exception.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.PRECONDITION)); + try (Connection connection = DriverManager.getConnection(jdbcUrl, USERNAME, PASSWORD); + Statement statement = connection.createStatement()) { + softly.assertThat(metadataTables(connection)).isEmpty(); + statement.execute("DROP VIEW hzb_status_page_org"); + } + assertRejectsPostgresqlObjects(jdbcUrl, kind, provisioner, target, softly); + assertRejectsContractWithoutHistory(jdbcUrl, kind, provisioner, target, softly); + softly.assertAll(); + } + + private static void assertRejectsContractWithoutHistory( + String jdbcUrl, + MetadataDatabaseKind kind, + TargetSchemaProvisioner provisioner, + MetadataDatabaseConfiguration target, + SoftAssertions softly) throws Exception { + try (Connection connection = DriverManager.getConnection(jdbcUrl, USERNAME, PASSWORD)) { + new TargetSchemaContract(kind).record(connection, Set.of()); + } + softly.assertThatThrownBy(() -> provisioner.provision(target)) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, exception -> + softly.assertThat(exception.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.PRECONDITION)); + try (Connection connection = DriverManager.getConnection(jdbcUrl, USERNAME, PASSWORD); + Statement statement = connection.createStatement()) { + statement.execute("DROP TABLE flyway_schema_contract"); + } + } + + private static void assertRejectsPostgresqlObjects( + String jdbcUrl, + MetadataDatabaseKind kind, + TargetSchemaProvisioner provisioner, + MetadataDatabaseConfiguration target, + SoftAssertions softly) throws Exception { + if (kind != MetadataDatabaseKind.POSTGRESQL) { + return; + } + assertRejectsObject(jdbcUrl, provisioner, target, softly, + "CREATE MATERIALIZED VIEW hzb_schema_probe AS SELECT 1 AS id", + "DROP MATERIALIZED VIEW hzb_schema_probe"); + assertRejectsObject(jdbcUrl, provisioner, target, softly, + "CREATE SEQUENCE hzb_schema_probe_sequence", + "DROP SEQUENCE hzb_schema_probe_sequence"); + } + + private static void assertRejectsObject( + String jdbcUrl, + TargetSchemaProvisioner provisioner, + MetadataDatabaseConfiguration target, + SoftAssertions softly, + String createSql, + String dropSql) throws Exception { + try (Connection connection = DriverManager.getConnection(jdbcUrl, USERNAME, PASSWORD); + Statement statement = connection.createStatement()) { + statement.execute(createSql); + } + softly.assertThatThrownBy(() -> provisioner.provision(target)) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, exception -> + softly.assertThat(exception.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.PRECONDITION)); + try (Connection connection = DriverManager.getConnection(jdbcUrl, USERNAME, PASSWORD); + Statement statement = connection.createStatement()) { + statement.execute(dropSql); + } + } + + private static void verify(String jdbcUrl, MetadataDatabaseKind kind, String dialect) throws Exception { + MetadataDatabaseConfiguration target = + new MetadataDatabaseConfiguration(kind, jdbcUrl, USERNAME, PASSWORD); + TargetSchemaProvisioner provisioner = new FlywayTargetSchemaProvisioner(); + assertProvisioningLogsAreSanitized(provisioner, target); + assertCurrentBaselineAllowsAdditionalTable(provisioner, target); + assertCurrentBaselineRejectsSchemaCorruption(provisioner, target); + + MetadataSchemaSnapshot baseline; + try (Connection connection = DriverManager.getConnection(jdbcUrl, USERNAME, PASSWORD)) { + assertThat(metadataTables(connection)).containsExactlyInAnyOrderElementsOf( + TargetSchemaBaselineResourceTest.mappedTables()); + try (Statement statement = connection.createStatement(); + ResultSet history = statement.executeQuery( + "SELECT version, type, success FROM flyway_schema_history ORDER BY installed_rank")) { + assertThat(history.next()).isTrue(); + assertThat(history.getString("version")).isEqualTo("206"); + assertThat(history.getString("type")).isEqualTo("SQL_BASELINE"); + assertThat(history.getBoolean("success")).isTrue(); + assertThat(history.next()).isFalse(); + } + baseline = MetadataSchemaSnapshot.capture(connection); + } + assertStandardFlywayAcceptsBaseline(jdbcUrl, kind); + validateHibernateMappings(jdbcUrl, dialect); + try (Connection connection = DriverManager.getConnection(jdbcUrl, USERNAME, PASSWORD)) { + replaceEarlyMigrationIndexWithIncorrectDefinition(connection, kind); + } + HistoricalMetadataSchema.rebuild(jdbcUrl, USERNAME, PASSWORD, kind.value()); + assertThat(historyRows(jdbcUrl)) + .extracting(HistoryRow::version) + .containsExactly("159", "160", "170", "172", "173", "180", "181", + "200", "201", "202", "203", "204", "205", "206"); + try (Connection connection = DriverManager.getConnection(jdbcUrl, USERNAME, PASSWORD)) { + MetadataSchemaSnapshot migrated = MetadataSchemaSnapshot.capture(connection); + assertThat(migrated.indexes()) + .filteredOn(index -> index.table().equals("hzb_monitor") + && index.name().equals("idx_hzb_monitor_app")) + .singleElement() + .extracting(MetadataSchemaSnapshot.Index::columns) + .isEqualTo(List.of(new MetadataSchemaSnapshot.IndexColumn((short) 1, "app", "a"))); + assertThat(schemaDifferences(baseline, migrated)).isEmpty(); + } + } + + private static List schemaDifferences( + MetadataSchemaSnapshot baseline, MetadataSchemaSnapshot migrated) { + List differences = new ArrayList<>(); + addDifferences(differences, "column", baseline.columns(), migrated.columns()); + addDifferences(differences, "primary key", baseline.primaryKeys(), migrated.primaryKeys()); + addDifferences(differences, "index", baseline.indexes(), migrated.indexes()); + addDifferences(differences, "foreign key", baseline.foreignKeys(), migrated.foreignKeys()); + return List.copyOf(differences); + } + + private static void addDifferences( + List differences, String kind, Set baseline, Set migrated) { + baseline.stream() + .filter(value -> !migrated.contains(value)) + .map(value -> "baseline-only " + kind + ": " + value) + .forEach(differences::add); + migrated.stream() + .filter(value -> !baseline.contains(value)) + .map(value -> "migration-only " + kind + ": " + value) + .forEach(differences::add); + } + + private static void replaceEarlyMigrationIndexWithIncorrectDefinition( + Connection connection, MetadataDatabaseKind kind) throws Exception { + try (Statement statement = connection.createStatement()) { + if (kind == MetadataDatabaseKind.MYSQL) { + statement.execute("DROP INDEX idx_hzb_monitor_app ON hzb_monitor"); + } else { + statement.execute("DROP INDEX idx_hzb_monitor_app"); + } + statement.execute("CREATE INDEX idx_hzb_monitor_app ON hzb_monitor(name)"); + } + } + + private static void assertCurrentBaselineAllowsAdditionalTable( + TargetSchemaProvisioner provisioner, MetadataDatabaseConfiguration target) throws Exception { + try (Connection connection = DriverManager.getConnection( + target.jdbcUrl(), target.username(), target.password()); + Statement statement = connection.createStatement()) { + statement.execute("CREATE TABLE unrelated_table (id INTEGER NOT NULL PRIMARY KEY)"); + try { + provisioner.provision(target); + } finally { + statement.execute("DROP TABLE unrelated_table"); + } + } + } + + private static void assertCurrentBaselineRejectsSchemaCorruption( + TargetSchemaProvisioner provisioner, MetadataDatabaseConfiguration target) throws Exception { + assertCorruptionRejected(provisioner, target, + target.kind() == MetadataDatabaseKind.MYSQL + ? "ALTER TABLE hzb_account MODIFY COLUMN username VARCHAR(63) NOT NULL" + : "ALTER TABLE hzb_account ALTER COLUMN username TYPE VARCHAR(63)", + target.kind() == MetadataDatabaseKind.MYSQL + ? "ALTER TABLE hzb_account MODIFY COLUMN username VARCHAR(64) NOT NULL" + : "ALTER TABLE hzb_account ALTER COLUMN username TYPE VARCHAR(64)"); + assertCorruptionRejected(provisioner, target, + target.kind() == MetadataDatabaseKind.MYSQL + ? "ALTER TABLE hzb_account MODIFY COLUMN credential_version INTEGER NOT NULL" + : "ALTER TABLE hzb_account ALTER COLUMN credential_version TYPE INTEGER", + target.kind() == MetadataDatabaseKind.MYSQL + ? "ALTER TABLE hzb_account MODIFY COLUMN credential_version BIGINT NOT NULL" + : "ALTER TABLE hzb_account ALTER COLUMN credential_version TYPE BIGINT"); + assertCorruptionRejected(provisioner, target, + target.kind() == MetadataDatabaseKind.MYSQL + ? "DROP INDEX idx_hzb_monitor_app ON hzb_monitor" + : "DROP INDEX idx_hzb_monitor_app", + "CREATE INDEX idx_hzb_monitor_app ON hzb_monitor(app)"); + assertCorruptionRejected(provisioner, target, + target.kind() == MetadataDatabaseKind.MYSQL + ? "ALTER TABLE hzb_ai_message DROP FOREIGN KEY fk_hzb_ai_message_conversation" + : "ALTER TABLE hzb_ai_message DROP CONSTRAINT fk_hzb_ai_message_conversation", + "ALTER TABLE hzb_ai_message ADD CONSTRAINT fk_hzb_ai_message_conversation " + + "FOREIGN KEY (conversation_id) REFERENCES hzb_ai_conversation(id)"); + assertCorruptionRejected(provisioner, target, + target.kind() == MetadataDatabaseKind.MYSQL + ? "ALTER TABLE hzb_ai_message DROP FOREIGN KEY fk_hzb_ai_message_conversation, " + + "ADD CONSTRAINT fk_hzb_ai_message_conversation_cascade " + + "FOREIGN KEY (conversation_id) " + + "REFERENCES hzb_ai_conversation(id) ON DELETE CASCADE" + : "ALTER TABLE hzb_ai_message DROP CONSTRAINT fk_hzb_ai_message_conversation, " + + "ADD CONSTRAINT fk_hzb_ai_message_conversation_cascade " + + "FOREIGN KEY (conversation_id) " + + "REFERENCES hzb_ai_conversation(id) ON DELETE CASCADE", + target.kind() == MetadataDatabaseKind.MYSQL + ? "ALTER TABLE hzb_ai_message " + + "DROP FOREIGN KEY fk_hzb_ai_message_conversation_cascade, " + + "ADD CONSTRAINT fk_hzb_ai_message_conversation FOREIGN KEY (conversation_id) " + + "REFERENCES hzb_ai_conversation(id)" + : "ALTER TABLE hzb_ai_message " + + "DROP CONSTRAINT fk_hzb_ai_message_conversation_cascade, " + + "ADD CONSTRAINT fk_hzb_ai_message_conversation FOREIGN KEY (conversation_id) " + + "REFERENCES hzb_ai_conversation(id)"); + } + + private static void assertCorruptionRejected( + TargetSchemaProvisioner provisioner, + MetadataDatabaseConfiguration target, + String corruptSql, + String restoreSql) throws Exception { + try (Connection connection = DriverManager.getConnection( + target.jdbcUrl(), target.username(), target.password()); + Statement statement = connection.createStatement()) { + statement.execute(corruptSql); + try { + assertThatThrownBy(() -> provisioner.provision(target)) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, exception -> + assertThat(exception.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.PRECONDITION)); + } finally { + statement.execute(restoreSql); + } + } + } + + private static void assertProvisioningLogsAreSanitized( + TargetSchemaProvisioner provisioner, MetadataDatabaseConfiguration target) throws Exception { + Logger root = (Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME); + ListAppender captured = new ListAppender<>(); + captured.start(); + root.addAppender(captured); + try { + provisioner.provision(target); + try (Connection connection = DriverManager.getConnection( + target.jdbcUrl(), target.username(), target.password())) { + TargetSchemaBaseline baseline = TargetSchemaBaseline.load(target.kind()); + assertThat(new TargetSchemaContract(target.kind()).matches(connection, baseline.expectedTables())) + .isTrue(); + } + provisioner.provision(target); + } finally { + root.detachAppender(captured); + captured.stop(); + } + assertThat(captured.list.stream().map(ILoggingEvent::getFormattedMessage).toList().toString()) + .doesNotContain(target.jdbcUrl(), target.password(), "CREATE TABLE", "INSERT INTO"); + } + + private static void assertStandardFlywayAcceptsBaseline(String jdbcUrl, MetadataDatabaseKind kind) + throws Exception { + List before = historyRows(jdbcUrl); + String vendor = kind == MetadataDatabaseKind.MYSQL ? "mysql" : "postgresql"; + Flyway flyway = Flyway.configure() + .dataSource(jdbcUrl, USERNAME, PASSWORD) + .locations("classpath:db/migration/" + vendor) + .cleanDisabled(true) + .validateMigrationNaming(true) + .load(); + flyway.validate(); + flyway.migrate(); + assertThat(historyRows(jdbcUrl)).isEqualTo(before); + } + + private static List historyRows(String jdbcUrl) throws Exception { + List rows = new ArrayList<>(); + try (Connection connection = DriverManager.getConnection(jdbcUrl, USERNAME, PASSWORD); + Statement statement = connection.createStatement(); + ResultSet result = statement.executeQuery( + "SELECT installed_rank, version, description, type, script, checksum, " + + "installed_by, installed_on, execution_time, success " + + "FROM flyway_schema_history ORDER BY installed_rank")) { + while (result.next()) { + rows.add(new HistoryRow( + result.getInt("installed_rank"), + result.getString("version"), + result.getString("description"), + result.getString("type"), + result.getString("script"), + result.getInt("checksum"), + result.getString("installed_by"), + result.getTimestamp("installed_on").toInstant(), + result.getInt("execution_time"), + result.getBoolean("success"))); + } + } + return List.copyOf(rows); + } + + private record HistoryRow( + int installedRank, + String version, + String description, + String type, + String script, + int checksum, + String installedBy, + java.time.Instant installedOn, + int executionTime, + boolean success) { + } + + private static Set metadataTables(Connection connection) throws Exception { + Set tables = new HashSet<>(); + DatabaseMetaData metadata = connection.getMetaData(); + try (ResultSet result = metadata.getTables(connection.getCatalog(), null, "hzb_%", new String[]{"TABLE"})) { + while (result.next()) { + tables.add(result.getString("TABLE_NAME").toLowerCase(Locale.ROOT)); + } + } + return tables; + } + + private static void validateHibernateMappings(String jdbcUrl, String dialect) throws Exception { + StandardServiceRegistryBuilder registryBuilder = new StandardServiceRegistryBuilder() + .applySetting("jakarta.persistence.jdbc.url", jdbcUrl) + .applySetting("jakarta.persistence.jdbc.user", USERNAME) + .applySetting("jakarta.persistence.jdbc.password", PASSWORD) + .applySetting("hibernate.dialect", dialect) + .applySetting("hibernate.physical_naming_strategy", + "org.hibernate.boot.model.naming.CamelCaseToUnderscoresNamingStrategy") + .applySetting("hibernate.hbm2ddl.auto", "validate"); + StandardServiceRegistry registry = registryBuilder.build(); + try { + MetadataSources sources = new MetadataSources(registry); + ClassPathScanningCandidateComponentProvider scanner = + new ClassPathScanningCandidateComponentProvider(false); + scanner.addIncludeFilter(new AnnotationTypeFilter(Entity.class)); + scanner.findCandidateComponents("org.apache.hertzbeat").stream() + .map(definition -> definition.getBeanClassName()) + .map(TargetSchemaProvisionerDatabaseTest::loadClass) + .forEach(sources::addAnnotatedClass); + try (SessionFactory ignored = sources.buildMetadata().buildSessionFactory()) { + assertThat(ignored.getMetamodel().getEntities()).hasSize(48); + } + } finally { + StandardServiceRegistryBuilder.destroy(registry); + } + } + + private static Class loadClass(String className) { + try { + return Class.forName(className); + } catch (ClassNotFoundException exception) { + throw new IllegalStateException("Mapped entity class is unavailable", exception); + } + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisionerMetadataFailureTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisionerMetadataFailureTest.java new file mode 100644 index 0000000000..95672a7772 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisionerMetadataFailureTest.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.DriverPropertyInfo; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Logger; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.api.Test; + +class TargetSchemaProvisionerMetadataFailureTest { + + @Test + void metadataProviderFailureIsSanitizedAtProvisioningBoundary() throws Exception { + String jdbcUrl = "jdbc:metadata-failure://private.example.test/hertzbeat?password=secret-value"; + TargetSchemaBaseline baseline = TargetSchemaBaseline.load(MetadataDatabaseKind.MYSQL); + Connection connection = currentSchemaConnection(baseline); + Driver driver = new TestConnectionDriver(jdbcUrl, connection); + DriverManager.registerDriver(driver); + try { + MetadataDatabaseConfiguration target = new MetadataDatabaseConfiguration( + MetadataDatabaseKind.MYSQL, jdbcUrl, "operator", "secret-value"); + + assertThatThrownBy(() -> new FlywayTargetSchemaProvisioner().provision(target)) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, exception -> { + assertThat(exception.failure()).isEqualTo(new TargetSchemaProvisioningFailure( + TargetSchemaProvisioningFailure.Phase.PRECONDITION, + TargetSchemaBaseline.VERSION, + "58000", + 777)); + assertThat(exception).hasNoCause(); + assertThat(exception.getMessage()) + .doesNotContain(jdbcUrl, "secret-value", "raw metadata diagnostic"); + }); + } finally { + DriverManager.deregisterDriver(driver); + } + } + + private static Connection currentSchemaConnection(TargetSchemaBaseline baseline) throws Exception { + Connection connection = mock(Connection.class); + DatabaseMetaData metadata = mock(DatabaseMetaData.class); + Statement historyStatement = mock(Statement.class); + ResultSet tables = tableRows(baseline); + ResultSet history = historyRow(baseline); + when(connection.getMetaData()).thenReturn(metadata); + when(connection.createStatement()).thenReturn(historyStatement); + when(metadata.getTables(isNull(), isNull(), anyString(), any(String[].class))) + .thenReturn(tables); + when(historyStatement.executeQuery(anyString())).thenReturn(history); + when(metadata.getColumns(isNull(), isNull(), anyString(), isNull())) + .thenThrow(new SQLException("raw metadata diagnostic", "58000", 777)); + return connection; + } + + private static ResultSet tableRows(TargetSchemaBaseline baseline) throws Exception { + List tables = new ArrayList<>(baseline.expectedTables()); + tables.add("flyway_schema_history"); + tables.add(TargetSchemaContract.TABLE); + AtomicInteger row = new AtomicInteger(-1); + ResultSet result = mock(ResultSet.class); + when(result.next()).thenAnswer(ignored -> row.incrementAndGet() < tables.size()); + when(result.getString("TABLE_NAME")).thenAnswer(ignored -> tables.get(row.get())); + return result; + } + + private static ResultSet historyRow(TargetSchemaBaseline baseline) throws Exception { + ResultSet result = mock(ResultSet.class); + when(result.next()).thenReturn(true, false); + when(result.getInt("installed_rank")).thenReturn(1); + when(result.getString("version")).thenReturn(TargetSchemaBaseline.VERSION); + when(result.getString("type")).thenReturn(TargetSchemaBaseline.TYPE); + when(result.getString("script")).thenReturn(TargetSchemaBaseline.SCRIPT); + when(result.getInt("checksum")).thenReturn(baseline.checksum()); + when(result.getBoolean("success")).thenReturn(true); + return result; + } + + private record TestConnectionDriver(String acceptedUrl, Connection connection) implements Driver { + + @Override + public Connection connect(String url, Properties info) { + return acceptsURL(url) ? connection : null; + } + + @Override + public boolean acceptsURL(String url) { + return acceptedUrl.equals(url); + } + + @Override + public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) { + return new DriverPropertyInfo[0]; + } + + @Override + public int getMajorVersion() { + return 1; + } + + @Override + public int getMinorVersion() { + return 0; + } + + @Override + public boolean jdbcCompliant() { + return false; + } + + @Override + public Logger getParentLogger() { + return Logger.getAnonymousLogger(); + } + } +} diff --git a/hertzbeat-startup/src/test/resources/db/historical/mysql/V159__schema.sql b/hertzbeat-startup/src/test/resources/db/historical/mysql/V159__schema.sql new file mode 100644 index 0000000000..56a2aa51e9 --- /dev/null +++ b/hertzbeat-startup/src/test/resources/db/historical/mysql/V159__schema.sql @@ -0,0 +1,593 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0. +-- +-- Immutable V159 schema fixture for migration-chain tests. +-- Do not derive this fixture from the current baseline or later migrations. + + create table hzb_ai_conversation ( + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + creator varchar(255), + modifier varchar(255), + title varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_ai_message ( + conversation_id bigint, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + creator varchar(255), + modifier varchar(255), + role varchar(255), + content longtext not null, + primary key (id) + ) engine=InnoDB; + + create table hzb_alert_define ( + enable bit not null, + period integer, + times integer, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + app varchar(255), + metric varchar(255), + field varchar(255), + preset bit, + priority integer, + tags varchar(255), + datasource varchar(100), + name varchar(100) not null, + expr varchar(2048), + labels varchar(2048), + template varchar(2048), + annotations varchar(4096), + creator varchar(255), + modifier varchar(255), + type varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_alert_define_monitor_bind ( + alert_define_id bigint, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + monitor_id bigint, + primary key (id) + ) engine=InnoDB; + + create table hzb_alert_group ( + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + common_labels varchar(2048), + group_key varchar(2048) character set ascii, + group_labels varchar(2048), + alert_fingerprints varchar(255), + common_annotations varchar(255), + creator varchar(255), + modifier varchar(255), + status varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_alert_group_converge ( + enable bit, + gmt_create datetime(6), + gmt_update datetime(6), + group_interval bigint, + group_wait bigint, + id bigint not null auto_increment, + repeat_interval bigint, + name varchar(100) not null, + group_labels varchar(1024), + creator varchar(255), + modifier varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_alert_inhibit ( + enable bit, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + name varchar(100) not null, + equal_labels varchar(2048), + source_labels varchar(2048), + target_labels varchar(2048), + creator varchar(255), + modifier varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_alert_silence ( + enable bit not null, + match_all bit not null, + times integer, + type tinyint not null, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + period_end datetime(6), + period_start datetime(6), + name varchar(100) not null, + labels varchar(2048), + creator varchar(255), + days varchar(255), + modifier varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_alert_single ( + trigger_times integer, + active_at bigint, + end_at bigint, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + start_at bigint, + fingerprint varchar(2048) character set ascii, + labels varchar(2048), + annotations varchar(4096), + content varchar(4096), + creator varchar(255), + modifier varchar(255), + status varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_bulletin ( + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + fields varchar(4096), + monitor_ids varchar(4096), + app varchar(255), + creator varchar(255), + modifier varchar(255), + name varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_collector ( + status tinyint not null, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + creator varchar(255), + ip varchar(255) not null, + mode varchar(255), + modifier varchar(255), + name varchar(255) not null, + version varchar(255), + + primary key (id), + check ((status>=0)) + ) engine=InnoDB; + + create table hzb_collector_monitor_bind ( + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + monitor_id bigint, + collector varchar(255), + creator varchar(255), + modifier varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_config ( + gmt_create datetime(6), + gmt_update datetime(6), + + content varchar(8192), + creator varchar(255), + modifier varchar(255), + type varchar(255) not null, + primary key (type) + ) engine=InnoDB; + + create table hzb_define ( + gmt_create datetime(6), + gmt_update datetime(6), + app varchar(255) not null, + creator varchar(255), + modifier varchar(255), + content longtext, + primary key (app) + ) engine=InnoDB; + + create table hzb_grafana_dashboard ( + enabled bit not null, + monitor_id bigint not null, + version bigint, + folder_uid varchar(255), + slug varchar(255), + status varchar(255), + uid varchar(255), + url varchar(255), + primary key (monitor_id) + ) engine=InnoDB; + + create table hzb_history ( + dou float(53), + int32 integer, + metric_type tinyint, + id bigint not null auto_increment, + time bigint, + str varchar(2048), + app varchar(255), + instance varchar(5000), + metric varchar(255), + metrics varchar(255), + monitor_id bigint, + primary key (id) + ) engine=InnoDB; + + create table hzb_metrics_favorite ( + create_time datetime(6), + id bigint not null auto_increment, + monitor_id bigint not null, + creator varchar(255) not null, + metrics_name varchar(255) not null, + primary key (id) + ) engine=InnoDB; + + create table hzb_monitor ( + intervals integer, + status tinyint not null, + type tinyint not null, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null, + job_id bigint, + schedule_type varchar(20), + app varchar(100), + cron_expression varchar(100), + host varchar(100), + name varchar(100), + scrape varchar(100), + annotations varchar(4096), + labels varchar(4096), + creator varchar(255), + description varchar(255), + modifier varchar(255), + primary key (id), + check ((status<=4) and (status>=0)) + ) engine=InnoDB; + + create table hzb_monitor_bind ( + biz_id bigint, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + monitor_id bigint, + creator varchar(255), + key_str varchar(255), + modifier varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_notice_receiver ( + agent_id integer, + lark_receive_type tinyint, + type tinyint not null, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + smn_ak varchar(22), + smn_project_id varchar(32), + smn_region varchar(32), + smn_sk varchar(42), + email varchar(100), + name varchar(100) not null, + phone varchar(100), + access_token varchar(300), + discord_bot_token varchar(300), + discord_channel_id varchar(300), + gotify_token varchar(300), + hook_auth_token varchar(300), + hook_auth_type varchar(300), + server_chan_token varchar(300), + slack_web_hook_url varchar(300), + smn_topic_urn varchar(300), + wechat_id varchar(300), + hook_url varchar(1000), + app_id varchar(255), + app_secret varchar(255), + chat_id varchar(255), + corp_id varchar(255), + creator varchar(255), + modifier varchar(255), + party_id varchar(255), + tag_id varchar(255), + tg_bot_token varchar(255), + tg_message_thread_id varchar(255), + tg_user_id varchar(255), + user_id varchar(255), + primary key (id), + check ((type>=0)) + ) engine=InnoDB; + + create table hzb_notice_rule ( + enable bit not null, + filter_all bit not null, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + period_end datetime(6), + period_start datetime(6), + template_id bigint, + name varchar(100) not null, + template_name varchar(100), + labels varchar(2048), + creator varchar(255), + days varchar(255), + modifier varchar(255), + receiver_id varchar(255) not null, + receiver_name varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_notice_template ( + preset boolean default false, + type tinyint not null, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + name varchar(100) not null, + creator varchar(255), + modifier varchar(255), + content text not null, + primary key (id), + check ((type>=0)) + ) engine=InnoDB; + + create table hzb_param ( + type tinyint not null, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + monitor_id bigint, + field varchar(100) not null, + param_value varchar(8126), + primary key (id), + check ((type>=0)) + ) engine=InnoDB; + + create table hzb_param_define ( + hide bit not null, + param_limit smallint, + required bit not null, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + name varchar(2048), + param_options varchar(2048), + app varchar(255), + creator varchar(255), + default_value varchar(255), + depend varchar(255), + field varchar(255), + key_alias varchar(255), + modifier varchar(255), + param_range varchar(255), + placeholder varchar(255), + type varchar(255), + value_alias varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_plugin_item ( + id bigint not null auto_increment, + metadata_id bigint, + class_identifier varchar(255), + type enum ('POST_ALERT','POST_COLLECT'), + primary key (id) + ) engine=InnoDB; + + create table hzb_plugin_metadata ( + enable_status bit, + param_count integer, + gmt_create datetime(6), + id bigint not null auto_increment, + creator varchar(255), + jar_file_path varchar(255), + name varchar(255) not null, + primary key (id) + ) engine=InnoDB; + + create table hzb_plugin_param ( + type tinyint not null, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + plugin_metadata_id bigint not null, + field varchar(100) not null, + param_value varchar(8126), + primary key (id), + check ((type>=0)) + ) engine=InnoDB; + + create table hzb_push_metrics ( + id bigint not null auto_increment, + monitor_id bigint, + time bigint, + metrics varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_status_page_component ( + config_state tinyint not null, + method tinyint not null, + state tinyint not null, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + org_id bigint, + labels varchar(4096), + creator varchar(255), + description varchar(255), + modifier varchar(255), + name varchar(255) not null, + primary key (id) + ) engine=InnoDB; + + create table hzb_status_page_history ( + abnormal integer, + normal integer, + state tinyint not null, + unknowing integer, + uptime float(53), + component_id bigint, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + timestamp bigint, + creator varchar(255), + modifier varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_status_page_incident ( + state tinyint not null, + end_time bigint, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + org_id bigint, + start_time bigint, + creator varchar(255), + modifier varchar(255), + name varchar(255) not null, + primary key (id) + ) engine=InnoDB; + + create table hzb_status_page_incident_component_bind ( + component_id bigint, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + incident_id bigint, + primary key (id) + ) engine=InnoDB; + + create table hzb_status_page_incident_content ( + state tinyint not null, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + incident_id bigint, + timestamp bigint, + creator varchar(255), + message varchar(255) not null, + modifier varchar(255), + primary key (id) + ) engine=InnoDB; + + create table hzb_status_page_org ( + state tinyint not null, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + color varchar(255), + creator varchar(255), + description varchar(255) not null, + feedback varchar(255), + home varchar(255) not null, + logo varchar(255) not null, + modifier varchar(255), + name varchar(255) not null, + primary key (id) + ) engine=InnoDB; + + create table hzb_tag ( + type tinyint, + gmt_create datetime(6), + gmt_update datetime(6), + id bigint not null auto_increment, + tag_value varchar(2048), + creator varchar(255), + description varchar(255), + modifier varchar(255), + name varchar(255) not null, + primary key (id), + check ((type<=3) and (type>=0)) + ) engine=InnoDB; + + + create index idx_message_conversation_id + on hzb_ai_message (conversation_id); + + create index index_alert_define_monitor + on hzb_alert_define_monitor_bind (alert_define_id, monitor_id); + + alter table hzb_alert_group + add constraint unique_group_key unique (group_key); + + create index idx_name + on hzb_alert_group_converge (name); + + alter table hzb_alert_single + add constraint unique_fingerprint unique (fingerprint); + + alter table hzb_collector + add constraint uk_hzb_collector_name unique (name); + + create index index_collector_monitor + on hzb_collector_monitor_bind (collector, monitor_id); + + + + + + create index history_query_index + on hzb_history (monitor_id, app, metrics, metric); + + alter table hzb_metrics_favorite + add constraint uk_hzb_metrics_favorite unique (creator, monitor_id, metrics_name); + + create index monitor_query_index + on hzb_monitor (app, host, name); + + create index index_monitor_bin + on hzb_monitor_bind (monitor_id); + + create index idx_hzb_param_monitor_id + on hzb_param (monitor_id); + + alter table hzb_param + add constraint uk_hzb_param_monitor_field unique (monitor_id, field); + + create index idx_hzb_plugin_param_plugin_metadata_id + on hzb_plugin_param (plugin_metadata_id); + + alter table hzb_plugin_param + add constraint uk_hzb_plugin_param_metadata_field unique (plugin_metadata_id, field); + + create index push_query_index + on hzb_push_metrics (monitor_id, time); + + create index index_incident_component + on hzb_status_page_incident_component_bind (incident_id); + + alter table hzb_ai_message + add constraint fk_hzb_ai_message_conversation + foreign key (conversation_id) + references hzb_ai_conversation (id); + + alter table hzb_plugin_item + add constraint fk_hzb_plugin_item_metadata + foreign key (metadata_id) + references hzb_plugin_metadata (id); + + alter table hzb_status_page_incident_content + add constraint fk_hzb_incident_content_incident + foreign key (incident_id) + references hzb_status_page_incident (id); diff --git a/hertzbeat-startup/src/test/resources/db/historical/postgresql/V159__schema.sql b/hertzbeat-startup/src/test/resources/db/historical/postgresql/V159__schema.sql new file mode 100644 index 0000000000..ab11072b07 --- /dev/null +++ b/hertzbeat-startup/src/test/resources/db/historical/postgresql/V159__schema.sql @@ -0,0 +1,573 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0. +-- +-- Immutable V159 schema fixture for migration-chain tests. +-- Do not derive this fixture from the current baseline or later migrations. + + create table hzb_ai_conversation ( + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + creator varchar(255), + modifier varchar(255), + title varchar(255), + primary key (id) + ); + + create table hzb_ai_message ( + conversation_id bigint, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + creator varchar(255), + modifier varchar(255), + role varchar(255), + content oid not null, + primary key (id) + ); + + create table hzb_alert_define ( + enable boolean not null, + period integer, + times integer, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + app varchar(255), + metric varchar(255), + field varchar(255), + preset boolean, + priority integer, + tags varchar(255), + datasource varchar(100), + name varchar(100) not null, + expr varchar(2048), + labels varchar(2048), + template varchar(2048), + annotations varchar(4096), + creator varchar(255), + modifier varchar(255), + type varchar(255), + primary key (id) + ); + + create table hzb_alert_define_monitor_bind ( + alert_define_id bigint, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + monitor_id bigint, + primary key (id) + ); + + create table hzb_alert_group ( + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + common_labels varchar(2048), + group_key varchar(2048), + group_labels varchar(2048), + alert_fingerprints varchar(255), + common_annotations varchar(255), + creator varchar(255), + modifier varchar(255), + status varchar(255), + primary key (id), + constraint unique_group_key unique (group_key) + ); + + create table hzb_alert_group_converge ( + enable boolean, + gmt_create timestamp(6), + gmt_update timestamp(6), + group_interval bigint, + group_wait bigint, + id bigint generated by default as identity, + repeat_interval bigint, + name varchar(100) not null, + group_labels varchar(1024), + creator varchar(255), + modifier varchar(255), + primary key (id) + ); + + create table hzb_alert_inhibit ( + enable boolean, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + name varchar(100) not null, + equal_labels varchar(2048), + source_labels varchar(2048), + target_labels varchar(2048), + creator varchar(255), + modifier varchar(255), + primary key (id) + ); + + create table hzb_alert_silence ( + enable boolean not null, + match_all boolean not null, + times integer, + type smallint not null, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + period_end timestamp(6) with time zone, + period_start timestamp(6) with time zone, + name varchar(100) not null, + labels varchar(2048), + creator varchar(255), + days varchar(255), + modifier varchar(255), + primary key (id) + ); + + create table hzb_alert_single ( + trigger_times integer, + active_at bigint, + end_at bigint, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + start_at bigint, + fingerprint varchar(2048), + labels varchar(2048), + annotations varchar(4096), + content varchar(4096), + creator varchar(255), + modifier varchar(255), + status varchar(255), + primary key (id), + constraint unique_fingerprint unique (fingerprint) + ); + + create table hzb_bulletin ( + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + fields varchar(4096), + monitor_ids varchar(4096), + app varchar(255), + creator varchar(255), + modifier varchar(255), + name varchar(255), + primary key (id) + ); + + create table hzb_collector ( + status smallint not null check ((status>=0)), + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + creator varchar(255), + ip varchar(255) not null, + mode varchar(255), + modifier varchar(255), + name varchar(255) not null, + version varchar(255), + + primary key (id), + unique (name) + ); + + create table hzb_collector_monitor_bind ( + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + monitor_id bigint, + collector varchar(255), + creator varchar(255), + modifier varchar(255), + primary key (id) + ); + + create table hzb_config ( + gmt_create timestamp(6), + gmt_update timestamp(6), + + content varchar(8192), + creator varchar(255), + modifier varchar(255), + type varchar(255) not null, + primary key (type) + ); + + create table hzb_define ( + gmt_create timestamp(6), + gmt_update timestamp(6), + app varchar(255) not null, + creator varchar(255), + modifier varchar(255), + content oid, + primary key (app) + ); + + create table hzb_grafana_dashboard ( + enabled boolean not null, + monitor_id bigint not null, + version bigint, + folder_uid varchar(255), + slug varchar(255), + status varchar(255), + uid varchar(255), + url varchar(255), + primary key (monitor_id) + ); + + create table hzb_history ( + dou float(53), + int32 integer, + metric_type smallint, + id bigint generated by default as identity, + time bigint, + str varchar(2048), + app varchar(255), + instance varchar(5000), + metric varchar(255), + metrics varchar(255), + monitor_id bigint, + primary key (id) + ); + + create table hzb_metrics_favorite ( + create_time timestamp(6), + id bigint generated by default as identity, + monitor_id bigint not null, + creator varchar(255) not null, + metrics_name varchar(255) not null, + primary key (id), + unique (creator, monitor_id, metrics_name) + ); + + create table hzb_monitor ( + intervals integer, + status smallint not null check ((status<=4) and (status>=0)), + type smallint not null, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint not null, + job_id bigint, + schedule_type varchar(20), + app varchar(100), + cron_expression varchar(100), + host varchar(100), + name varchar(100), + scrape varchar(100), + annotations varchar(4096), + labels varchar(4096), + creator varchar(255), + description varchar(255), + modifier varchar(255), + primary key (id) + ); + + create table hzb_monitor_bind ( + biz_id bigint, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + monitor_id bigint, + creator varchar(255), + key_str varchar(255), + modifier varchar(255), + primary key (id) + ); + + create table hzb_notice_receiver ( + agent_id integer, + lark_receive_type smallint, + type smallint not null check ((type>=0)), + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + smn_ak varchar(22), + smn_project_id varchar(32), + smn_region varchar(32), + smn_sk varchar(42), + email varchar(100), + name varchar(100) not null, + phone varchar(100), + access_token varchar(300), + discord_bot_token varchar(300), + discord_channel_id varchar(300), + gotify_token varchar(300), + hook_auth_token varchar(300), + hook_auth_type varchar(300), + server_chan_token varchar(300), + slack_web_hook_url varchar(300), + smn_topic_urn varchar(300), + wechat_id varchar(300), + hook_url varchar(1000), + app_id varchar(255), + app_secret varchar(255), + chat_id varchar(255), + corp_id varchar(255), + creator varchar(255), + modifier varchar(255), + party_id varchar(255), + tag_id varchar(255), + tg_bot_token varchar(255), + tg_message_thread_id varchar(255), + tg_user_id varchar(255), + user_id varchar(255), + primary key (id) + ); + + create table hzb_notice_rule ( + enable boolean not null, + filter_all boolean not null, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + period_end timestamp(6) with time zone, + period_start timestamp(6) with time zone, + template_id bigint, + name varchar(100) not null, + template_name varchar(100), + labels varchar(2048), + creator varchar(255), + days varchar(255), + modifier varchar(255), + receiver_id varchar(255) not null, + receiver_name varchar(255), + primary key (id) + ); + + create table hzb_notice_template ( + preset boolean default false, + type smallint not null check ((type>=0)), + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + name varchar(100) not null, + creator varchar(255), + modifier varchar(255), + content oid not null, + primary key (id) + ); + + create table hzb_param ( + type smallint not null check ((type>=0)), + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + monitor_id bigint, + field varchar(100) not null, + param_value varchar(8126), + primary key (id), + constraint uk_hzb_param_monitor_field unique (monitor_id, field) + ); + + create table hzb_param_define ( + hide boolean not null, + param_limit smallint, + required boolean not null, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + name varchar(2048), + param_options varchar(2048), + app varchar(255), + creator varchar(255), + default_value varchar(255), + depend varchar(255), + field varchar(255), + key_alias varchar(255), + modifier varchar(255), + param_range varchar(255), + placeholder varchar(255), + type varchar(255), + value_alias varchar(255), + primary key (id) + ); + + create table hzb_plugin_item ( + id bigint generated by default as identity, + metadata_id bigint, + class_identifier varchar(255), + type varchar(255) check ((type in ('POST_ALERT','POST_COLLECT'))), + primary key (id) + ); + + create table hzb_plugin_metadata ( + enable_status boolean, + param_count integer, + gmt_create timestamp(6), + id bigint generated by default as identity, + creator varchar(255), + jar_file_path varchar(255), + name varchar(255) not null, + primary key (id) + ); + + create table hzb_plugin_param ( + type smallint not null check ((type>=0)), + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + plugin_metadata_id bigint not null, + field varchar(100) not null, + param_value varchar(8126), + primary key (id), + constraint uk_hzb_plugin_param_metadata_field unique (plugin_metadata_id, field) + ); + + create table hzb_push_metrics ( + id bigint generated by default as identity, + monitor_id bigint, + time bigint, + metrics varchar(255), + primary key (id) + ); + + create table hzb_status_page_component ( + config_state smallint not null, + method smallint not null, + state smallint not null, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + org_id bigint, + labels varchar(4096), + creator varchar(255), + description varchar(255), + modifier varchar(255), + name varchar(255) not null, + primary key (id) + ); + + create table hzb_status_page_history ( + abnormal integer, + normal integer, + state smallint not null, + unknowing integer, + uptime float(53), + component_id bigint, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + timestamp bigint, + creator varchar(255), + modifier varchar(255), + primary key (id) + ); + + create table hzb_status_page_incident ( + state smallint not null, + end_time bigint, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + org_id bigint, + start_time bigint, + creator varchar(255), + modifier varchar(255), + name varchar(255) not null, + primary key (id) + ); + + create table hzb_status_page_incident_component_bind ( + component_id bigint, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + incident_id bigint, + primary key (id) + ); + + create table hzb_status_page_incident_content ( + state smallint not null, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + incident_id bigint, + timestamp bigint, + creator varchar(255), + message varchar(255) not null, + modifier varchar(255), + primary key (id) + ); + + create table hzb_status_page_org ( + state smallint not null, + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + color varchar(255), + creator varchar(255), + description varchar(255) not null, + feedback varchar(255), + home varchar(255) not null, + logo varchar(255) not null, + modifier varchar(255), + name varchar(255) not null, + primary key (id) + ); + + create table hzb_tag ( + type smallint check ((type<=3) and (type>=0)), + gmt_create timestamp(6), + gmt_update timestamp(6), + id bigint generated by default as identity, + tag_value varchar(2048), + creator varchar(255), + description varchar(255), + modifier varchar(255), + name varchar(255) not null, + primary key (id) + ); + + create index idx_message_conversation_id + on hzb_ai_message (conversation_id); + + create index index_alert_define_monitor + on hzb_alert_define_monitor_bind (alert_define_id, monitor_id); + + create index idx_name + on hzb_alert_group_converge (name); + + create index index_collector_monitor + on hzb_collector_monitor_bind (collector, monitor_id); + + + + + + create index history_query_index + on hzb_history (monitor_id, app, metrics, metric); + + create index monitor_query_index + on hzb_monitor (app, host, name); + + create index index_monitor_bin + on hzb_monitor_bind (monitor_id); + + create index idx_hzb_param_monitor_id + on hzb_param (monitor_id); + + create index idx_hzb_plugin_param_plugin_metadata_id + on hzb_plugin_param (plugin_metadata_id); + + create index push_query_index + on hzb_push_metrics (monitor_id, time); + + create index index_incident_component + on hzb_status_page_incident_component_bind (incident_id); + + alter table if exists hzb_ai_message + add constraint fk_hzb_ai_message_conversation + foreign key (conversation_id) + references hzb_ai_conversation; + + alter table if exists hzb_plugin_item + add constraint fk_hzb_plugin_item_metadata + foreign key (metadata_id) + references hzb_plugin_metadata; + + alter table if exists hzb_status_page_incident_content + add constraint fk_hzb_incident_content_incident + foreign key (incident_id) + references hzb_status_page_incident; From fe70a592b95e3cd42aadd82290ae8d93e2cf281d Mon Sep 17 00:00:00 2001 From: Logic Date: Sun, 9 Aug 2026 20:35:42 +0800 Subject: [PATCH 33/71] Clarify metadata migration admission --- .../setup/api/DeploymentApiContract.java | 30 +++++-- .../setup/api/DeploymentController.java | 9 ++ .../setup/api/MigrationContractValidator.java | 77 ++++++++++++++--- .../setup/api/OperationIdValidator.java | 33 ++++++++ .../workflow/MetadataMigrationPolicy.java | 16 +--- .../setup/api/DeploymentApiContractTest.java | 83 ++++++++++++++++--- .../setup/api/DeploymentControllerTest.java | 52 ++++++++++++ .../setup/api/OperationIdValidatorTest.java | 41 +++++++++ .../workflow/MetadataMigrationPolicyTest.java | 46 ++++++++-- 9 files changed, 333 insertions(+), 54 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/OperationIdValidator.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/OperationIdValidatorTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContract.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContract.java index 7d69d2d357..7729fa81b7 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContract.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContract.java @@ -64,6 +64,14 @@ public final class DeploymentApiContract { ACTIVE } + /** How a future migration coordinator can satisfy the maintenance precondition. */ + public enum MaintenanceAdmission implements WireValue { + USE_CURRENT, + AUTO_ENTER, + UNAVAILABLE, + NOT_APPLICABLE + } + /** Deployment shape relevant to migration safety. */ public enum DeploymentTopology implements WireValue { SINGLE_NODE, @@ -130,18 +138,28 @@ public final class DeploymentApiContract { } /** Explicit migration eligibility and safe blocker for the deployment screen. */ - public record MigrationCapability(boolean allowed, SetupErrorCode blockedBy) { + public record MigrationCapability( + boolean allowed, + SetupErrorCode blockedBy, + @NotNull MaintenanceAdmission maintenanceAdmission, + String activeOperationId) { public MigrationCapability { - MigrationContractValidator.validateCapability(allowed, blockedBy); + MigrationContractValidator.validateCapability( + allowed, blockedBy, maintenanceAdmission, activeOperationId); } - public static MigrationCapability permitted() { - return new MigrationCapability(true, null); + public static MigrationCapability permitted(MaintenanceAdmission admission) { + return new MigrationCapability(true, null, admission, null); } - public static MigrationCapability blocked(SetupErrorCode blocker) { - return new MigrationCapability(false, blocker); + public static MigrationCapability blocked(SetupErrorCode blocker, MaintenanceAdmission admission) { + return blocked(blocker, admission, null); + } + + public static MigrationCapability blocked( + SetupErrorCode blocker, MaintenanceAdmission admission, String activeOperationId) { + return new MigrationCapability(false, blocker, admission, activeOperationId); } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentController.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentController.java index e23e1c8d72..40f51dbc22 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentController.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentController.java @@ -70,6 +70,7 @@ public final class DeploymentController { @GetMapping(DeploymentApiContract.MIGRATION_OPERATION_PATH) public ResponseEntity migration(@PathVariable String operationId) { + requireOperationId(operationId); MigrationView migration = workflow().migration(operationId); if (migration == null) { throw new SetupApiException(SetupApiContract.SetupErrorCode.OPERATION_NOT_FOUND, HttpStatus.NOT_FOUND); @@ -80,12 +81,14 @@ public final class DeploymentController { @PostMapping(DeploymentApiContract.ACTIVATE_PATH) public ResponseEntity activate( @PathVariable String operationId, @Valid @RequestBody ActivateMigrationRequest request) { + requireOperationId(operationId); return SetupHttpContract.noStore().body(workflow().activate(operationId, request)); } @PostMapping(DeploymentApiContract.EXPORT_PATH) public ResponseEntity export( @PathVariable String operationId, @Valid @RequestBody MigrationExportRequest request) { + requireOperationId(operationId); MigrationExportRenderer renderer = renderer(); ExportResponse metadata = workflow().prepareExport(operationId, request); StreamingResponseBody body = output -> renderer.write(operationId, request, output); @@ -103,6 +106,12 @@ public final class DeploymentController { return workflow; } + private void requireOperationId(String operationId) { + if (!OperationIdValidator.isSafe(operationId)) { + throw new SetupApiException(SetupApiContract.SetupErrorCode.INVALID_REQUEST, HttpStatus.BAD_REQUEST); + } + } + private MigrationExportRenderer renderer() { MigrationExportRenderer renderer = rendererProvider.getIfUnique(); if (renderer == null) { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/MigrationContractValidator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/MigrationContractValidator.java index d0b3aeff57..a5c4e4b93f 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/MigrationContractValidator.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/MigrationContractValidator.java @@ -20,6 +20,7 @@ package org.apache.hertzbeat.manager.setup.api; import java.time.Instant; import java.util.Set; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentTopology; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MaintenanceAdmission; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MaintenanceMode; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationCapability; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; @@ -43,16 +44,29 @@ final class MigrationContractValidator { SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED, SetupErrorCode.MIGRATION_MULTI_NODE_UNSUPPORTED, SetupErrorCode.MIGRATION_TOPOLOGY_UNAVAILABLE, - SetupErrorCode.MIGRATION_MAINTENANCE_REQUIRED); + SetupErrorCode.MIGRATION_MAINTENANCE_REQUIRED, + SetupErrorCode.MIGRATION_UNAVAILABLE, + SetupErrorCode.OPERATION_CONFLICT); private MigrationContractValidator() { } - static void validateCapability(boolean allowed, SetupErrorCode blockedBy) { + static void validateCapability( + boolean allowed, SetupErrorCode blockedBy, + MaintenanceAdmission admission, String activeOperationId) { if (allowed != (blockedBy == null) - || (blockedBy != null && !CAPABILITY_BLOCKERS.contains(blockedBy))) { + || admission == null + || blockedBy != null && !CAPABILITY_BLOCKERS.contains(blockedBy)) { invalid("Migration capability and blocker are inconsistent"); } + boolean validAdmission = switch (admission) { + case USE_CURRENT, AUTO_ENTER -> allowed && activeOperationId == null; + case NOT_APPLICABLE -> !allowed && structuralBlocker(blockedBy) && activeOperationId == null; + case UNAVAILABLE -> unavailable(blockedBy, activeOperationId); + }; + if (!validAdmission) { + invalid("Migration maintenance admission is inconsistent"); + } } static void validateTarget(MigrationTarget target, MetadataDatabaseConfiguration database) { @@ -68,15 +82,16 @@ final class MigrationContractValidator { invalid("Deployment migration context is incomplete"); } if (database.kind() != MetadataDatabaseKind.H2) { - requireBlocker(capability, SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED); + requireBlocker(capability, SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED, + MaintenanceAdmission.NOT_APPLICABLE); } else if (topology == DeploymentTopology.MULTI_NODE) { - requireBlocker(capability, SetupErrorCode.MIGRATION_MULTI_NODE_UNSUPPORTED); + requireBlocker(capability, SetupErrorCode.MIGRATION_MULTI_NODE_UNSUPPORTED, + MaintenanceAdmission.NOT_APPLICABLE); } else if (topology == DeploymentTopology.UNKNOWN) { - requireBlocker(capability, SetupErrorCode.MIGRATION_TOPOLOGY_UNAVAILABLE); - } else if (maintenance == MaintenanceMode.INACTIVE) { - requireBlocker(capability, SetupErrorCode.MIGRATION_MAINTENANCE_REQUIRED); - } else if (!capability.allowed()) { - invalid("Active single-node H2 migration must be permitted"); + requireBlocker(capability, SetupErrorCode.MIGRATION_TOPOLOGY_UNAVAILABLE, + MaintenanceAdmission.NOT_APPLICABLE); + } else { + validateSingleNodeAdmission(maintenance, capability); } } @@ -86,7 +101,7 @@ final class MigrationContractValidator { int progress, Instant createdAt, Instant startedAt, Instant completedAt, VerificationState verification, SetupErrorCode errorCode, long pollAfterMillis, boolean activationAvailable, boolean restartRequired, boolean externalApplyRequired) { - if (operationId == null || operationId.isBlank() || source != MetadataDatabaseKind.H2 || target == null + if (!OperationIdValidator.isSafe(operationId) || source != MetadataDatabaseKind.H2 || target == null || state == null || stage == null || createdAt == null || verification == null || progress < 0 || progress > 100 || pollAfterMillis < 0) { @@ -180,8 +195,44 @@ final class MigrationContractValidator { || state == MigrationOperationState.ROLLED_BACK; } - private static void requireBlocker(MigrationCapability capability, SetupErrorCode expected) { - if (capability.allowed() || capability.blockedBy() != expected) { + private static void validateSingleNodeAdmission( + MaintenanceMode maintenance, MigrationCapability capability) { + if (capability.allowed()) { + MaintenanceAdmission expected = maintenance == MaintenanceMode.ACTIVE + ? MaintenanceAdmission.USE_CURRENT : MaintenanceAdmission.AUTO_ENTER; + if (capability.maintenanceAdmission() != expected) { + invalid("Migration admission does not match maintenance state"); + } + return; + } + if (structuralBlocker(capability.blockedBy())) { + invalid("Single-node migration cannot report a structural blocker"); + } + if (capability.blockedBy() == SetupErrorCode.MIGRATION_MAINTENANCE_REQUIRED + && maintenance != MaintenanceMode.INACTIVE) { + invalid("Maintenance-required blocker is stale"); + } + } + + private static boolean structuralBlocker(SetupErrorCode blocker) { + return blocker == SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED + || blocker == SetupErrorCode.MIGRATION_MULTI_NODE_UNSUPPORTED + || blocker == SetupErrorCode.MIGRATION_TOPOLOGY_UNAVAILABLE; + } + + private static boolean unavailable(SetupErrorCode blocker, String activeOperationId) { + if (blocker == SetupErrorCode.OPERATION_CONFLICT) { + return OperationIdValidator.isSafe(activeOperationId); + } + return (blocker == SetupErrorCode.MIGRATION_UNAVAILABLE + || blocker == SetupErrorCode.MIGRATION_MAINTENANCE_REQUIRED) + && activeOperationId == null; + } + + private static void requireBlocker( + MigrationCapability capability, SetupErrorCode expected, MaintenanceAdmission admission) { + if (capability.allowed() || capability.blockedBy() != expected + || capability.maintenanceAdmission() != admission || capability.activeOperationId() != null) { invalid("Deployment migration blocker does not match its structure"); } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/OperationIdValidator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/OperationIdValidator.java new file mode 100644 index 0000000000..c2524295b6 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/OperationIdValidator.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import java.util.regex.Pattern; + +/** Shared path-segment boundary for stable operation identifiers exposed by setup APIs. */ +final class OperationIdValidator { + + private static final Pattern URL_SAFE_ID = Pattern.compile("[A-Za-z0-9][A-Za-z0-9._-]{0,127}"); + + private OperationIdValidator() { + } + + static boolean isSafe(String operationId) { + return operationId != null && URL_SAFE_ID.matcher(operationId).matches(); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationPolicy.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationPolicy.java index 557040ec23..da41f75da9 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationPolicy.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationPolicy.java @@ -17,14 +17,11 @@ package org.apache.hertzbeat.manager.setup.workflow; -import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentTopology; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentView; -import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MaintenanceMode; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationView; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.TargetInspection; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; import org.apache.hertzbeat.manager.setup.api.SetupApiException; import org.springframework.http.HttpStatus; @@ -37,17 +34,8 @@ public final class MetadataMigrationPolicy { if (deployment == null || target == null || targetInspection == null) { throw new SetupApiException(SetupErrorCode.INVALID_REQUEST, HttpStatus.BAD_REQUEST); } - if (deployment.managementDatabase().kind() != MetadataDatabaseKind.H2) { - throw conflict(SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED); - } - if (deployment.topology() == DeploymentTopology.UNKNOWN) { - throw conflict(SetupErrorCode.MIGRATION_TOPOLOGY_UNAVAILABLE); - } - if (deployment.topology() == DeploymentTopology.MULTI_NODE) { - throw conflict(SetupErrorCode.MIGRATION_MULTI_NODE_UNSUPPORTED); - } - if (deployment.maintenanceMode() != MaintenanceMode.ACTIVE) { - throw conflict(SetupErrorCode.MIGRATION_MAINTENANCE_REQUIRED); + if (!deployment.migration().allowed()) { + throw conflict(deployment.migration().blockedBy()); } if (targetInspection == TargetInspection.UNKNOWN) { throw conflict(SetupErrorCode.METADATA_CONNECTION_FAILED); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContractTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContractTest.java index d7d9333f4c..039994bbb8 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContractTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContractTest.java @@ -31,6 +31,7 @@ import java.util.Arrays; import java.util.List; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentTopology; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MaintenanceMode; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MaintenanceAdmission; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationCapability; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; @@ -64,7 +65,8 @@ class DeploymentApiContractTest { DeploymentApiContract.EXPORT_PATH); assertComponents(DeploymentApiContract.DeploymentView.class, "observedAt", "managementDatabase", "greptimeDatabase", "applyMode", "maintenanceMode", "topology", "migration"); - assertComponents(DeploymentApiContract.MigrationCapability.class, "allowed", "blockedBy"); + assertComponents(DeploymentApiContract.MigrationCapability.class, "allowed", "blockedBy", + "maintenanceAdmission", "activeOperationId"); assertComponents(DeploymentApiContract.MetadataMigrationValidationRequest.class, "target", "targetDatabase"); assertComponents(DeploymentApiContract.MetadataMigrationRequest.class, "target", "targetDatabase", "applyMode"); @@ -76,6 +78,8 @@ class DeploymentApiContractTest { assertComponents(DeploymentApiContract.MigrationExportRequest.class, "format", "expectedState", "targetDatabase"); assertWireValues(MaintenanceMode.values(), "inactive", "active"); + assertWireValues(MaintenanceAdmission.values(), + "use_current", "auto_enter", "unavailable", "not_applicable"); assertWireValues(DeploymentTopology.values(), "single_node", "multi_node", "unknown"); assertWireValues(MigrationTarget.values(), "mysql", "postgresql"); assertWireValues(MigrationStage.values(), "queued", "copying", "verifying", "ready_to_activate", @@ -101,30 +105,77 @@ class DeploymentApiContractTest { new ManagementDatabaseSummary(MetadataDatabaseKind.H2, true, ConfigSource.UI_MANAGED, false), new TelemetryStoreSummary(TelemetryStoreKind.GREPTIME, true, ConfigSource.UI_MANAGED, false), ApplyMode.MANAGED_WRITE, MaintenanceMode.ACTIVE, DeploymentTopology.SINGLE_NODE, - MigrationCapability.permitted()); + MigrationCapability.permitted(MaintenanceAdmission.USE_CURRENT)); assertTrue(view.migration().allowed()); assertNull(view.migration().blockedBy()); + assertEquals(MaintenanceAdmission.USE_CURRENT, view.migration().maintenanceAdmission()); + assertNull(view.migration().activeOperationId()); String json = objectMapper.writeValueAsString(view); + assertTrue(json.contains("\"maintenanceAdmission\":\"use_current\"")); + assertTrue(json.contains("\"activeOperationId\":null")); assertFalse(json.contains("jdbc:")); assertFalse(json.contains("password")); assertFalse(json.contains("table")); assertThrows(IllegalArgumentException.class, - () -> new MigrationCapability(false, null)); + () -> new MigrationCapability(false, null, MaintenanceAdmission.UNAVAILABLE, null)); assertThrows(IllegalArgumentException.class, - () -> new MigrationCapability(true, SetupErrorCode.MIGRATION_MULTI_NODE_UNSUPPORTED)); + () -> new MigrationCapability(true, SetupErrorCode.MIGRATION_MULTI_NODE_UNSUPPORTED, + MaintenanceAdmission.USE_CURRENT, null)); assertDeploymentRejected(MetadataDatabaseKind.MYSQL, DeploymentTopology.SINGLE_NODE, - MigrationCapability.permitted()); + MigrationCapability.permitted(MaintenanceAdmission.USE_CURRENT)); assertDeploymentRejected(MetadataDatabaseKind.H2, DeploymentTopology.MULTI_NODE, - MigrationCapability.blocked(SetupErrorCode.MIGRATION_TOPOLOGY_UNAVAILABLE)); + MigrationCapability.blocked(SetupErrorCode.MIGRATION_TOPOLOGY_UNAVAILABLE, + MaintenanceAdmission.NOT_APPLICABLE)); assertDeploymentRejected(MetadataDatabaseKind.H2, DeploymentTopology.UNKNOWN, - MigrationCapability.blocked(SetupErrorCode.MIGRATION_MULTI_NODE_UNSUPPORTED)); + MigrationCapability.blocked(SetupErrorCode.MIGRATION_MULTI_NODE_UNSUPPORTED, + MaintenanceAdmission.NOT_APPLICABLE)); assertThrows(IllegalArgumentException.class, - () -> MigrationCapability.blocked(SetupErrorCode.CONFIG_READ_ONLY)); + () -> MigrationCapability.blocked(SetupErrorCode.CONFIG_READ_ONLY, + MaintenanceAdmission.UNAVAILABLE)); assertDeploymentRejected(MetadataDatabaseKind.H2, MaintenanceMode.INACTIVE, - DeploymentTopology.SINGLE_NODE, MigrationCapability.permitted()); + DeploymentTopology.SINGLE_NODE, + MigrationCapability.permitted(MaintenanceAdmission.USE_CURRENT)); assertDoesNotThrow(() -> deployment(MaintenanceMode.INACTIVE, - MigrationCapability.blocked(SetupErrorCode.MIGRATION_MAINTENANCE_REQUIRED))); + MigrationCapability.blocked(SetupErrorCode.MIGRATION_MAINTENANCE_REQUIRED, + MaintenanceAdmission.UNAVAILABLE))); + } + + @Test + void migrationCapabilityMatchesTheFrozenMaintenanceAdmissionMatrix() { + assertDoesNotThrow(() -> deployment(MetadataDatabaseKind.MYSQL, MaintenanceMode.ACTIVE, + DeploymentTopology.SINGLE_NODE, MigrationCapability.blocked( + SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED, MaintenanceAdmission.NOT_APPLICABLE))); + assertDoesNotThrow(() -> deployment(MetadataDatabaseKind.H2, MaintenanceMode.ACTIVE, + DeploymentTopology.MULTI_NODE, MigrationCapability.blocked( + SetupErrorCode.MIGRATION_MULTI_NODE_UNSUPPORTED, MaintenanceAdmission.NOT_APPLICABLE))); + assertDoesNotThrow(() -> deployment(MetadataDatabaseKind.H2, MaintenanceMode.INACTIVE, + DeploymentTopology.UNKNOWN, MigrationCapability.blocked( + SetupErrorCode.MIGRATION_TOPOLOGY_UNAVAILABLE, MaintenanceAdmission.NOT_APPLICABLE))); + assertDoesNotThrow(() -> deployment(MaintenanceMode.ACTIVE, MigrationCapability.blocked( + SetupErrorCode.OPERATION_CONFLICT, MaintenanceAdmission.UNAVAILABLE, "migration-42"))); + assertDoesNotThrow(() -> deployment(MaintenanceMode.INACTIVE, MigrationCapability.blocked( + SetupErrorCode.MIGRATION_UNAVAILABLE, MaintenanceAdmission.UNAVAILABLE))); + assertDoesNotThrow(() -> deployment(MaintenanceMode.ACTIVE, + MigrationCapability.permitted(MaintenanceAdmission.USE_CURRENT))); + assertDoesNotThrow(() -> deployment(MaintenanceMode.INACTIVE, + MigrationCapability.permitted(MaintenanceAdmission.AUTO_ENTER))); + assertDoesNotThrow(() -> deployment(MaintenanceMode.INACTIVE, MigrationCapability.blocked( + SetupErrorCode.MIGRATION_MAINTENANCE_REQUIRED, MaintenanceAdmission.UNAVAILABLE))); + + assertThrows(IllegalArgumentException.class, () -> deployment(MaintenanceMode.ACTIVE, + MigrationCapability.permitted(MaintenanceAdmission.AUTO_ENTER))); + assertThrows(IllegalArgumentException.class, () -> deployment(MaintenanceMode.ACTIVE, + MigrationCapability.blocked(SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED, + MaintenanceAdmission.NOT_APPLICABLE))); + assertThrows(IllegalArgumentException.class, () -> MigrationCapability.blocked( + SetupErrorCode.OPERATION_CONFLICT, MaintenanceAdmission.UNAVAILABLE, null)); + assertThrows(IllegalArgumentException.class, () -> MigrationCapability.blocked( + SetupErrorCode.MIGRATION_UNAVAILABLE, MaintenanceAdmission.UNAVAILABLE, "migration-42")); + assertThrows(IllegalArgumentException.class, () -> MigrationCapability.blocked( + SetupErrorCode.OPERATION_CONFLICT, MaintenanceAdmission.UNAVAILABLE, "../migration")); + assertThrows(IllegalArgumentException.class, () -> MigrationCapability.blocked( + SetupErrorCode.OPERATION_CONFLICT, MaintenanceAdmission.UNAVAILABLE, "..")); } @Test @@ -218,6 +269,8 @@ class DeploymentApiContractTest { assertThrows(IllegalArgumentException.class, () -> rolledBackView( SetupErrorCode.MIGRATION_ACTIVATION_FAILED, VerificationState.SUCCEEDED, 99)); assertThrows(IllegalArgumentException.class, () -> migrationViewWithIdentity(" ", MigrationTarget.MYSQL)); + assertThrows(IllegalArgumentException.class, + () -> migrationViewWithIdentity("migration/../secret", MigrationTarget.MYSQL)); assertThrows(IllegalArgumentException.class, () -> migrationViewWithIdentity("migration-1", null)); } @@ -269,10 +322,16 @@ class DeploymentApiContractTest { private DeploymentApiContract.DeploymentView deployment( MaintenanceMode maintenance, MigrationCapability capability) { + return deployment(MetadataDatabaseKind.H2, maintenance, DeploymentTopology.SINGLE_NODE, capability); + } + + private DeploymentApiContract.DeploymentView deployment( + MetadataDatabaseKind kind, MaintenanceMode maintenance, + DeploymentTopology topology, MigrationCapability capability) { return new DeploymentApiContract.DeploymentView(Instant.parse("2026-08-09T00:00:00Z"), - new ManagementDatabaseSummary(MetadataDatabaseKind.H2, true, ConfigSource.UI_MANAGED, false), + new ManagementDatabaseSummary(kind, true, ConfigSource.UI_MANAGED, false), new TelemetryStoreSummary(TelemetryStoreKind.GREPTIME, true, ConfigSource.UI_MANAGED, false), - ApplyMode.MANAGED_WRITE, maintenance, DeploymentTopology.SINGLE_NODE, capability); + ApplyMode.MANAGED_WRITE, maintenance, topology, capability); } private void assertComponents(Class type, String... names) { diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerTest.java index f9e77982b6..464d244f11 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerTest.java @@ -35,6 +35,11 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. import java.io.ByteArrayOutputStream; import java.time.Instant; import java.util.List; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentTopology; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentView; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MaintenanceAdmission; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MaintenanceMode; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationCapability; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationExportRequest; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; @@ -43,9 +48,14 @@ import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationVie import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportFormat; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigSource; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ManagementDatabaseSummary; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreSummary; import org.apache.hertzbeat.manager.setup.workflow.MigrationExportRenderer; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -77,6 +87,7 @@ class DeploymentControllerTest { @Test void routesValidationCreationPollingAndActivationWithoutStoringResponses() throws Exception { + when(workflow.deployment()).thenReturn(deployment()); when(workflow.validate(any())).thenReturn(new ValidationResponse( true, Instant.parse("2026-08-09T00:00:00Z"), null, List.of())); when(workflow.migrate(any())).thenReturn(readyMigration()); @@ -92,6 +103,12 @@ class DeploymentControllerTest { "applyMode":"managed_write"} """; + mvc.perform(get(DeploymentApiContract.DEPLOYMENT_PATH)) + .andExpect(status().isOk()).andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.migration.allowed").value(false)) + .andExpect(jsonPath("$.migration.blockedBy").value("operation_conflict")) + .andExpect(jsonPath("$.migration.maintenanceAdmission").value("unavailable")) + .andExpect(jsonPath("$.migration.activeOperationId").value("migration-42")); mvc.perform(post(DeploymentApiContract.VALIDATE_PATH).contentType(MediaType.APPLICATION_JSON) .content(target)) .andExpect(status().isOk()).andExpect(header().string("Cache-Control", "no-store")) @@ -139,6 +156,32 @@ class DeploymentControllerTest { .andExpect(jsonPath("$.errorCode").value("operation_not_found")); } + @Test + void rejectsInvalidOperationIdsBeforeWorkflowOrRendererDispatch() throws Exception { + mvc.perform(get(DeploymentApiContract.MIGRATION_OPERATION_PATH, ".hidden")) + .andExpect(status().isBadRequest()) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.errorCode").value("invalid_request")); + mvc.perform(post(DeploymentApiContract.ACTIVATE_PATH, "\u8fc1\u79fb") + .contentType(MediaType.APPLICATION_JSON) + .content("{\"expectedState\":\"ready_to_activate\"}")) + .andExpect(status().isBadRequest()) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.errorCode").value("invalid_request")); + String exportRequest = """ + {"format":"env","expectedState":"awaiting_external_apply", + "targetDatabase":{"kind":"mysql","jdbcUrl":"jdbc:mysql://db/hertzbeat", + "username":"operator","password":"export-secret"}} + """; + mvc.perform(post(DeploymentApiContract.EXPORT_PATH, "a".repeat(129)) + .contentType(MediaType.APPLICATION_JSON).content(exportRequest)) + .andExpect(status().isBadRequest()) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.errorCode").value("invalid_request")); + + verifyNoInteractions(workflow, exportRenderer); + } + @Test void missingWorkflowReturnsStableNoStoreUnavailable() throws Exception { assertDeploymentUnavailable(mvc(List.of(), List.of(exportRenderer))); @@ -203,6 +246,15 @@ class DeploymentControllerTest { VerificationState.SUCCEEDED, null, 0, true, false, false); } + private DeploymentView deployment() { + return new DeploymentView(Instant.parse("2026-08-09T00:00:00Z"), + new ManagementDatabaseSummary(MetadataDatabaseKind.H2, true, ConfigSource.UI_MANAGED, false), + new TelemetryStoreSummary(TelemetryStoreKind.GREPTIME, true, ConfigSource.UI_MANAGED, false), + ApplyMode.MANAGED_WRITE, MaintenanceMode.ACTIVE, DeploymentTopology.SINGLE_NODE, + MigrationCapability.blocked(SetupApiContract.SetupErrorCode.OPERATION_CONFLICT, + MaintenanceAdmission.UNAVAILABLE, "migration-42")); + } + private MigrationView restartingMigration() { return new MigrationView("migration-1", MigrationOperationState.AWAITING_RESTART, SetupApiContract.MetadataDatabaseKind.H2, MigrationTarget.MYSQL, diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/OperationIdValidatorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/OperationIdValidatorTest.java new file mode 100644 index 0000000000..fe37c9d533 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/OperationIdValidatorTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +/** Freezes operation identifiers to the deployment frontend path-segment contract. */ +class OperationIdValidatorTest { + + @Test + void acceptsOnlyFrontendCompatibleOperationIds() { + assertTrue(OperationIdValidator.isSafe("a")); + assertTrue(OperationIdValidator.isSafe("a.b_c-d")); + assertTrue(OperationIdValidator.isSafe("a".repeat(128))); + + assertFalse(OperationIdValidator.isSafe(".hidden")); + assertFalse(OperationIdValidator.isSafe("-id")); + assertFalse(OperationIdValidator.isSafe("_id")); + assertFalse(OperationIdValidator.isSafe("~id")); + assertFalse(OperationIdValidator.isSafe("a~b")); + assertFalse(OperationIdValidator.isSafe("a".repeat(129))); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationPolicyTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationPolicyTest.java index 2af6410c5d..48bad5b24b 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationPolicyTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationPolicyTest.java @@ -25,6 +25,7 @@ import java.time.Instant; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentTopology; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentView; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MaintenanceMode; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MaintenanceAdmission; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationCapability; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; @@ -49,14 +50,13 @@ class MetadataMigrationPolicyTest { private final MetadataMigrationPolicy policy = new MetadataMigrationPolicy(); @Test - void permitsOnlySingleNodeEmptyTargetMigrationFromH2() { + void permitsCurrentOrAtomicAutoEnterAdmissionForSingleNodeEmptyTargetMigrationFromH2() { assertDoesNotThrow(() -> policy.requireMigrationAllowed( deployment(MaintenanceMode.ACTIVE, DeploymentTopology.SINGLE_NODE), MigrationTarget.MYSQL, TargetInspection.EMPTY)); - assertFailure(SetupErrorCode.MIGRATION_MAINTENANCE_REQUIRED, - () -> policy.requireMigrationAllowed( - deployment(MaintenanceMode.INACTIVE, DeploymentTopology.SINGLE_NODE), - MigrationTarget.MYSQL, TargetInspection.EMPTY)); + assertDoesNotThrow(() -> policy.requireMigrationAllowed( + deployment(MaintenanceMode.INACTIVE, DeploymentTopology.SINGLE_NODE), + MigrationTarget.MYSQL, TargetInspection.EMPTY)); assertFailure(SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED, () -> policy.requireMigrationAllowed( deployment(MetadataDatabaseKind.MYSQL, MaintenanceMode.ACTIVE, @@ -78,6 +78,24 @@ class MetadataMigrationPolicyTest { () -> policy.requireMigrationAllowed( deployment(MaintenanceMode.ACTIVE, DeploymentTopology.MULTI_NODE), MigrationTarget.POSTGRESQL, TargetInspection.EMPTY)); + assertFailure(SetupErrorCode.OPERATION_CONFLICT, + () -> policy.requireMigrationAllowed(deployment(MaintenanceMode.ACTIVE, + DeploymentTopology.SINGLE_NODE, MigrationCapability.blocked( + SetupErrorCode.OPERATION_CONFLICT, + MaintenanceAdmission.UNAVAILABLE, "migration-42")), + MigrationTarget.MYSQL, TargetInspection.EMPTY)); + assertFailure(SetupErrorCode.MIGRATION_UNAVAILABLE, + () -> policy.requireMigrationAllowed(deployment(MaintenanceMode.INACTIVE, + DeploymentTopology.SINGLE_NODE, MigrationCapability.blocked( + SetupErrorCode.MIGRATION_UNAVAILABLE, + MaintenanceAdmission.UNAVAILABLE)), + MigrationTarget.MYSQL, TargetInspection.EMPTY)); + assertFailure(SetupErrorCode.MIGRATION_MAINTENANCE_REQUIRED, + () -> policy.requireMigrationAllowed(deployment(MaintenanceMode.INACTIVE, + DeploymentTopology.SINGLE_NODE, MigrationCapability.blocked( + SetupErrorCode.MIGRATION_MAINTENANCE_REQUIRED, + MaintenanceAdmission.UNAVAILABLE)), + MigrationTarget.MYSQL, TargetInspection.EMPTY)); } @Test @@ -106,11 +124,21 @@ class MetadataMigrationPolicyTest { case UNKNOWN -> SetupErrorCode.MIGRATION_TOPOLOGY_UNAVAILABLE; case SINGLE_NODE -> kind == MetadataDatabaseKind.H2 ? null : SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED; }; - if (blocker == null && maintenance == MaintenanceMode.INACTIVE) { - blocker = SetupErrorCode.MIGRATION_MAINTENANCE_REQUIRED; - } MigrationCapability capability = blocker == null - ? MigrationCapability.permitted() : MigrationCapability.blocked(blocker); + ? MigrationCapability.permitted(maintenance == MaintenanceMode.ACTIVE + ? MaintenanceAdmission.USE_CURRENT : MaintenanceAdmission.AUTO_ENTER) + : MigrationCapability.blocked(blocker, MaintenanceAdmission.NOT_APPLICABLE); + return deployment(maintenance, topology, capability, kind); + } + + private DeploymentView deployment( + MaintenanceMode maintenance, DeploymentTopology topology, MigrationCapability capability) { + return deployment(maintenance, topology, capability, MetadataDatabaseKind.H2); + } + + private DeploymentView deployment( + MaintenanceMode maintenance, DeploymentTopology topology, + MigrationCapability capability, MetadataDatabaseKind kind) { return new DeploymentView(Instant.parse("2026-08-09T00:00:00Z"), new ManagementDatabaseSummary(kind, true, ConfigSource.UI_MANAGED, false), new TelemetryStoreSummary(TelemetryStoreKind.GREPTIME, true, ConfigSource.UI_MANAGED, false), From b0ba220883d3c93a6c7ec7320a2a395e48d0fe08 Mon Sep 17 00:00:00 2001 From: Logic Date: Sun, 9 Aug 2026 22:12:41 +0800 Subject: [PATCH 34/71] Persist metadata migration operations --- .../setup/api/OperationIdValidator.java | 4 +- .../FileSetupTransitionIntentLock.java | 108 ------ .../FileSetupTransitionIntentStore.java | 8 +- ...CommittedSetupFileDurabilityException.java | 18 + .../setup/security/SecureSetupFile.java | 43 +++ .../setup/security/SecureSetupFileLock.java | 173 +++++++++ .../workflow/FileMigrationOperationStore.java | 182 ++++++++++ .../MigrationOperationCollectionPolicy.java | 43 +++ .../workflow/MigrationOperationFileCodec.java | 160 +++++++++ .../MigrationOperationFilePublisher.java | 35 ++ .../workflow/MigrationOperationSnapshot.java | 67 ++++ .../workflow/MigrationOperationStore.java | 25 ++ .../MigrationOperationStoreException.java | 26 ++ .../MigrationOperationTransitionPolicy.java | 113 ++++++ .../workflow/MigrationRollbackOrigin.java | 34 ++ .../SecureSetupFileLockProcessMain.java | 69 ++++ .../security/SecureSetupFileLockTest.java | 183 ++++++++++ .../setup/security/SecureSetupFileTest.java | 19 +- .../FileMigrationOperationStoreTest.java | 328 ++++++++++++++++++ ...igrationOperationTransitionPolicyTest.java | 227 ++++++++++++ .../runtime/SetupTransitionSplitLockTest.java | 5 +- 21 files changed, 1753 insertions(+), 117 deletions(-) delete mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/FileSetupTransitionIntentLock.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/CommittedSetupFileDurabilityException.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileLock.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationCollectionPolicy.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationFileCodec.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationFilePublisher.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationSnapshot.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationStore.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationStoreException.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicy.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationRollbackOrigin.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileLockProcessMain.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileLockTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStoreTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicyTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/OperationIdValidator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/OperationIdValidator.java index c2524295b6..e05138b47e 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/OperationIdValidator.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/OperationIdValidator.java @@ -20,14 +20,14 @@ package org.apache.hertzbeat.manager.setup.api; import java.util.regex.Pattern; /** Shared path-segment boundary for stable operation identifiers exposed by setup APIs. */ -final class OperationIdValidator { +public final class OperationIdValidator { private static final Pattern URL_SAFE_ID = Pattern.compile("[A-Za-z0-9][A-Za-z0-9._-]{0,127}"); private OperationIdValidator() { } - static boolean isSafe(String operationId) { + public static boolean isSafe(String operationId) { return operationId != null && URL_SAFE_ID.matcher(operationId).matches(); } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/FileSetupTransitionIntentLock.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/FileSetupTransitionIntentLock.java deleted file mode 100644 index a9a413d2b1..0000000000 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/FileSetupTransitionIntentLock.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hertzbeat.manager.setup.runtime; - -import java.io.IOException; -import java.nio.channels.FileChannel; -import java.nio.channels.FileLock; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.LinkOption; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.locks.ReentrantLock; -import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; - -/** Cooperative owner-only serialization for transition intent marker operations. */ -final class FileSetupTransitionIntentLock { - private static final String RELATIVE_PATH = "data/config/.setup-transition-intent.lock"; - private static final ConcurrentMap JVM_LOCKS = new ConcurrentHashMap<>(); - private final Path installationRoot; - private final Path lockFile; - private final ReentrantLock jvmLock; - private boolean initialized; - - FileSetupTransitionIntentLock(Path installationRoot) { - this(installationRoot, RELATIVE_PATH); - } - - FileSetupTransitionIntentLock(Path installationRoot, String relativePath) { - this.installationRoot = installationRoot.toAbsolutePath().normalize(); - lockFile = this.installationRoot.resolve(relativePath).normalize(); - if (!lockFile.startsWith(this.installationRoot)) { - throw new IllegalArgumentException("Setup transition intent lock must remain inside the installation root"); - } - // FileChannel.lock throws OverlappingFileLockException instead of waiting inside one JVM. - jvmLock = JVM_LOCKS.computeIfAbsent(lockFile, ignored -> new ReentrantLock(true)); - } - - T execute(IoOperation operation) throws IOException { - jvmLock.lock(); - try { - initialize(); - validate(); - try (FileChannel channel = FileChannel.open( - lockFile, Set.of(StandardOpenOption.WRITE, LinkOption.NOFOLLOW_LINKS)); - FileLock ignored = channel.lock()) { - return operation.run(); - } - } finally { - jvmLock.unlock(); - } - } - - void execute(IoAction action) throws IOException { - execute(() -> { - action.run(); - return null; - }); - } - - private void initialize() throws IOException { - if (initialized) { - return; - } - try { - SecureSetupFile.create(installationRoot, lockFile, new byte[] {'l', 'o', 'c', 'k', '\n'}); - } catch (FileAlreadyExistsException existing) { - // The monotonic marker protocol remains correct if deployment replaces this lock inode. - } - validate(); - SecureSetupFile.forceParentDirectoryIfSupported(installationRoot, lockFile); - initialized = true; - } - - private void validate() throws IOException { - if (!SecureSetupFile.existsInsideRootWithoutLinks(installationRoot, lockFile) - || !SecureSetupFile.isOwnerOnlyRegularFile(lockFile)) { - throw new IOException("Setup transition intent lock is invalid"); - } - } - - @FunctionalInterface - interface IoOperation { - T run() throws IOException; - } - - @FunctionalInterface - interface IoAction { - void run() throws IOException; - } -} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/FileSetupTransitionIntentStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/FileSetupTransitionIntentStore.java index dc59787e58..086c5716ee 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/FileSetupTransitionIntentStore.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/runtime/FileSetupTransitionIntentStore.java @@ -27,6 +27,7 @@ import java.util.Arrays; import java.util.Objects; import java.util.Optional; import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFileLock; /** Owner-only, monotonic managed-file adapter for pending setup runtime transitions. */ public final class FileSetupTransitionIntentStore implements SetupTransitionIntentStore { @@ -36,13 +37,14 @@ public final class FileSetupTransitionIntentStore implements SetupTransitionInte static final String TERMINAL_RELATIVE_PATH = "data/config/setup-transition-closed"; private static final String INSTALLATION_CLOSED = "INSTALLATION_CLOSED"; + private static final String LOCK_PATH = "data/config/.setup-transition-intent.lock"; private static final int MAXIMUM_BYTES = 64; private final Path installationRoot; private final Marker configurationMarker; private final Marker completionMarker; private final Marker terminalMarker; private final ParentDirectorySync parentDirectorySync; - private final FileSetupTransitionIntentLock intentLock; + private final SecureSetupFileLock intentLock; private final MarkerObservation markerObservation; public FileSetupTransitionIntentStore(Path installationRoot) { @@ -52,12 +54,12 @@ public final class FileSetupTransitionIntentStore implements SetupTransitionInte FileSetupTransitionIntentStore(Path installationRoot, ParentDirectorySync parentDirectorySync) { this(installationRoot, parentDirectorySync, - new FileSetupTransitionIntentLock(installationRoot), MarkerObservation.NONE); + new SecureSetupFileLock(installationRoot, LOCK_PATH), MarkerObservation.NONE); } FileSetupTransitionIntentStore( Path installationRoot, ParentDirectorySync parentDirectorySync, - FileSetupTransitionIntentLock intentLock, MarkerObservation markerObservation) { + SecureSetupFileLock intentLock, MarkerObservation markerObservation) { this.installationRoot = Objects.requireNonNull(installationRoot, "installationRoot") .toAbsolutePath().normalize(); configurationMarker = marker(RELATIVE_PATH, Intent.CONFIGURATION_APPLIED.name()); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/CommittedSetupFileDurabilityException.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/CommittedSetupFileDurabilityException.java new file mode 100644 index 0000000000..2ec4ba1ae7 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/CommittedSetupFileDurabilityException.java @@ -0,0 +1,18 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.security; + +import java.io.IOException; + +/** Rename committed the new file, but parent-directory durability could not be confirmed. */ +public final class CommittedSetupFileDurabilityException extends IOException { + + public CommittedSetupFileDurabilityException() { + super("Setup file committed with uncertain directory durability"); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFile.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFile.java index 728e88f3b2..a5772e52a1 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFile.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFile.java @@ -24,6 +24,7 @@ import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.FileAttribute; import java.util.Arrays; @@ -126,6 +127,43 @@ public final class SecureSetupFile { return true; } + static Path prepareTrustedRoot(Path trustedRoot) throws IOException { + Path absoluteRoot = absolute(trustedRoot); + if (!Files.exists(absoluteRoot, LinkOption.NOFOLLOW_LINKS)) { + createMissingParents(absoluteRoot); + } + return absoluteRoot.toRealPath(); + } + + /** + * Publishes an already-forced owner-only temporary file without exposing a partial replacement. + * Providers without atomic-move support fail closed; callers must not downgrade to a non-atomic move. + */ + public static void atomicReplace(Path trustedRoot, Path source, Path target) throws IOException { + atomicReplace(trustedRoot, source, target, + replaced -> forceParentDirectoryIfSupported(trustedRoot, replaced)); + } + + static void atomicReplace( + Path trustedRoot, Path source, Path target, ParentDirectorySync parentDirectorySync) throws IOException { + Path resolvedSource = resolveWithoutLinksInsideRoot(trustedRoot, source); + Path resolvedTarget = resolveWithoutLinksInsideRoot(trustedRoot, target); + if (!isOwnerOnlyRegularFile(resolvedSource) || !resolvedSource.getParent().equals(resolvedTarget.getParent())) { + throw new IOException("Setup replacement source is invalid"); + } + if (Files.exists(resolvedTarget, LinkOption.NOFOLLOW_LINKS) + && !isOwnerOnlyRegularFile(resolvedTarget)) { + throw new IOException("Setup replacement target is invalid"); + } + Files.move(resolvedSource, resolvedTarget, StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + try { + parentDirectorySync.force(target); + } catch (IOException failure) { + throw new CommittedSetupFileDurabilityException(); + } + } + private static byte[] readResolvedOwnerOnly(Path resolvedTarget, int maximumBytes) throws IOException { if (!isOwnerOnlyRegularFile(resolvedTarget)) { throw new IOException("Setup file is not an owner-only regular file"); @@ -226,4 +264,9 @@ public final class SecureSetupFile { private static Path absolute(Path path) { return path.toAbsolutePath().normalize(); } + + @FunctionalInterface + interface ParentDirectorySync { + void force(Path target) throws IOException; + } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileLock.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileLock.java new file mode 100644 index 0000000000..831af95c2c --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileLock.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.security; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.ReentrantLock; + +/** + * Cooperative cross-context and cross-process lock for secure setup-file transactions. + * A missing trusted root is created through the secure setup-file path boundary. Non-cooperating + * same-owner code must not unlink lock entries during a transaction; Java/NIO cannot exclude that + * actor portably. Identity checks detect replacement, while cooperating writers use the stable inode. + */ +public final class SecureSetupFileLock { + + private static final ConcurrentMap JVM_LOCKS = new ConcurrentHashMap<>(); + private static final ConcurrentMap JVM_IDENTITIES = new ConcurrentHashMap<>(); + private static final String IDENTITY_PREFIX = "secure-setup-lock-v1:"; + private static final int MAXIMUM_IDENTITY_BYTES = 80; + private final Path installationRoot; + private final Path lockFile; + private final ReentrantLock jvmLock; + + public SecureSetupFileLock(Path installationRoot, String relativePath) { + this.installationRoot = canonicalRoot(installationRoot); + lockFile = this.installationRoot.resolve(relativePath).normalize(); + if (!lockFile.startsWith(this.installationRoot)) { + throw new IllegalArgumentException("Secure setup-file lock must remain inside the installation root"); + } + // FileChannel.lock throws OverlappingFileLockException instead of waiting inside one JVM. + jvmLock = JVM_LOCKS.computeIfAbsent(lockFile, ignored -> new ReentrantLock(true)); + } + + public T execute(IoOperation operation) throws IOException { + jvmLock.lock(); + try { + LockIdentity identity = initializeAndValidate(); + try (FileChannel channel = FileChannel.open( + lockFile, Set.of(StandardOpenOption.READ, StandardOpenOption.WRITE, LinkOption.NOFOLLOW_LINKS)); + FileLock ignored = channel.lock()) { + validateLockedIdentity(channel, identity); + T result = operation.run(); + validateLockedIdentity(channel, identity); + return result; + } + } finally { + jvmLock.unlock(); + } + } + + public void execute(IoAction action) throws IOException { + execute(() -> { + action.run(); + return null; + }); + } + + private LockIdentity initializeAndValidate() throws IOException { + try { + String created = IDENTITY_PREFIX + UUID.randomUUID() + '\n'; + SecureSetupFile.create(installationRoot, lockFile, created.getBytes(StandardCharsets.UTF_8)); + } catch (FileAlreadyExistsException existing) { + // Cooperating contexts converge on the existing owner-only inode. + } + validate(); + SecureSetupFile.forceParentDirectoryIfSupported(installationRoot, lockFile); + String observed = readPathIdentity(); + String expected = JVM_IDENTITIES.putIfAbsent(lockFile, observed); + if (expected != null && !expected.equals(observed)) { + throw new IOException("Secure setup-file lock identity changed"); + } + return new LockIdentity(observed, readPathFileKey()); + } + + private void validate() throws IOException { + if (!SecureSetupFile.existsInsideRootWithoutLinks(installationRoot, lockFile) + || !SecureSetupFile.isOwnerOnlyRegularFile(lockFile)) { + throw new IOException("Secure setup-file lock is invalid"); + } + } + + private void validateLockedIdentity(FileChannel channel, LockIdentity expected) throws IOException { + validate(); + if (!expected.token().equals(readChannelIdentity(channel)) + || !Objects.equals(expected.fileKey(), readPathFileKey())) { + throw new IOException("Secure setup-file lock identity changed"); + } + } + + private Object readPathFileKey() throws IOException { + BasicFileAttributes attributes = Files.readAttributes( + lockFile, BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS); + if (attributes.fileKey() == null) { + throw new IOException("Secure setup-file lock identity is unavailable"); + } + return attributes.fileKey(); + } + + private String readPathIdentity() throws IOException { + byte[] encoded = SecureSetupFile.readOwnerOnlyWithoutLinks( + installationRoot, lockFile, MAXIMUM_IDENTITY_BYTES); + return validateIdentity(new String(encoded, StandardCharsets.UTF_8)); + } + + private String readChannelIdentity(FileChannel channel) throws IOException { + long size = channel.size(); + if (size <= 0 || size > MAXIMUM_IDENTITY_BYTES) { + throw new IOException("Secure setup-file lock identity is invalid"); + } + ByteBuffer encoded = ByteBuffer.allocate((int) size); + channel.position(0); + while (encoded.hasRemaining() && channel.read(encoded) >= 0) { + // Continue until the bounded identity is complete. + } + return validateIdentity(new String(encoded.array(), StandardCharsets.UTF_8)); + } + + private String validateIdentity(String encoded) throws IOException { + String identity = encoded.strip(); + if (!identity.startsWith(IDENTITY_PREFIX)) { + throw new IOException("Secure setup-file lock identity is invalid"); + } + try { + UUID.fromString(identity.substring(IDENTITY_PREFIX.length())); + return identity; + } catch (IllegalArgumentException invalid) { + throw new IOException("Secure setup-file lock identity is invalid"); + } + } + + private static Path canonicalRoot(Path root) { + try { + return SecureSetupFile.prepareTrustedRoot(root); + } catch (IOException failure) { + throw new IllegalArgumentException("Secure setup-file root is unsafe"); + } + } + + private record LockIdentity(String token, Object fileKey) { + } + + /** I/O operation serialized by the cooperative lock. */ + @FunctionalInterface + public interface IoOperation { + T run() throws IOException; + } + + /** I/O action serialized by the cooperative lock. */ + @FunctionalInterface + public interface IoAction { + void run() throws IOException; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java new file mode 100644 index 0000000000..ca05277d90 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.OperationIdValidator; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.security.CommittedSetupFileDurabilityException; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFileLock; + +/** Root-bound owner-only file adapter for the single active migration operation. */ +public final class FileMigrationOperationStore implements MigrationOperationStore { + + static final String RELATIVE_PATH = "data/config/metadata-migration-operations"; + static final int HISTORY_LIMIT = 8; + private static final String LOCK_PATH = "data/config/.metadata-migration-operations.lock"; + private static final int MAXIMUM_BYTES = 64 * 1024; + private final Path installationRoot; + private final Path operationFile; + private final Publisher publisher; + private final SecureSetupFileLock lock; + private final MigrationOperationFileCodec codec = new MigrationOperationFileCodec(); + private final MigrationOperationCollectionPolicy collectionPolicy = new MigrationOperationCollectionPolicy(); + private final MigrationOperationTransitionPolicy transitionPolicy = new MigrationOperationTransitionPolicy(); + + public FileMigrationOperationStore(Path installationRoot) { + this(installationRoot, new MigrationOperationFilePublisher(normalize(installationRoot))); + } + + FileMigrationOperationStore(Path installationRoot, Publisher publisher) { + this.installationRoot = normalize(installationRoot); + operationFile = this.installationRoot.resolve(RELATIVE_PATH); + this.publisher = Objects.requireNonNull(publisher, "publisher"); + lock = new SecureSetupFileLock(this.installationRoot, LOCK_PATH); + } + + @Override + public MigrationOperationSnapshot create(MigrationOperationSnapshot snapshot) { + Objects.requireNonNull(snapshot, "snapshot"); + if (snapshot.state() != MigrationOperationState.PENDING) { + throw failure(SetupErrorCode.INVALID_REQUEST); + } + return locked(() -> { + List snapshots = read(); + if (snapshots.stream().anyMatch(value -> !value.terminal()) + || snapshots.stream().anyMatch(value -> value.operationId().equals(snapshot.operationId()))) { + throw failure(SetupErrorCode.OPERATION_CONFLICT); + } + snapshots.add(snapshot); + write(snapshots); + return snapshot; + }); + } + + @Override + public Optional find(String operationId) { + requireSafeId(operationId); + return locked(() -> read().stream() + .filter(snapshot -> snapshot.operationId().equals(operationId)).findFirst()); + } + + @Override + public List history() { + return locked(() -> List.copyOf(read())); + } + + @Override + public MigrationOperationSnapshot compareAndTransition( + String operationId, MigrationOperationState expectedState, MigrationOperationSnapshot replacement) { + requireSafeId(operationId); + Objects.requireNonNull(expectedState, "expectedState"); + Objects.requireNonNull(replacement, "replacement"); + return locked(() -> transition(read(), operationId, expectedState, replacement)); + } + + private MigrationOperationSnapshot transition( + List snapshots, String operationId, + MigrationOperationState expectedState, MigrationOperationSnapshot replacement) { + for (int index = 0; index < snapshots.size(); index++) { + MigrationOperationSnapshot current = snapshots.get(index); + if (current.operationId().equals(operationId)) { + if (current.state() != expectedState) { + throw failure(SetupErrorCode.OPERATION_CONFLICT); + } + transitionPolicy.requireAllowed(current, replacement); + snapshots.set(index, replacement); + trim(snapshots); + write(snapshots); + return replacement; + } + } + throw failure(SetupErrorCode.OPERATION_NOT_FOUND); + } + + private List read() { + if (!Files.exists(operationFile, LinkOption.NOFOLLOW_LINKS)) { + return new ArrayList<>(); + } + byte[] encoded = null; + try { + encoded = SecureSetupFile.readOwnerOnlyWithoutLinks( + installationRoot, operationFile, MAXIMUM_BYTES); + return new ArrayList<>(codec.decode(encoded)); + } catch (IOException | RuntimeException invalid) { + throw failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } finally { + if (encoded != null) { + Arrays.fill(encoded, (byte) 0); + } + } + } + + private void write(List snapshots) { + collectionPolicy.validate(snapshots); + byte[] encoded = codec.encode(snapshots); + try { + publisher.publish(operationFile, encoded); + } catch (CommittedSetupFileDurabilityException uncertain) { + throw failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } catch (IOException failure) { + throw failure(SetupErrorCode.CONFIG_WRITE_FAILED); + } finally { + Arrays.fill(encoded, (byte) 0); + } + } + + private void trim(List snapshots) { + while (snapshots.stream().filter(MigrationOperationSnapshot::terminal).count() > HISTORY_LIMIT) { + int oldestTerminal = -1; + for (int index = 0; index < snapshots.size() && oldestTerminal < 0; index++) { + if (snapshots.get(index).terminal()) { + oldestTerminal = index; + } + } + snapshots.remove(oldestTerminal); + } + } + + private T locked(SecureSetupFileLock.IoOperation operation) { + try { + return lock.execute(operation); + } catch (MigrationOperationStoreException failure) { + throw failure; + } catch (IOException failure) { + throw failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + } + + private void requireSafeId(String operationId) { + if (!OperationIdValidator.isSafe(operationId)) { + throw failure(SetupErrorCode.INVALID_REQUEST); + } + } + + private MigrationOperationStoreException failure(SetupErrorCode errorCode) { + return new MigrationOperationStoreException(errorCode); + } + + private static Path normalize(Path root) { + return Objects.requireNonNull(root, "installationRoot").toAbsolutePath().normalize(); + } + + @FunctionalInterface + interface Publisher { + void publish(Path target, byte[] content) throws IOException; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationCollectionPolicy.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationCollectionPolicy.java new file mode 100644 index 0000000000..ee52a1f45d --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationCollectionPolicy.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** Aggregate invariants for the one-active-operation plus bounded-history model. */ +final class MigrationOperationCollectionPolicy { + + void validate(List snapshots) { + Set operationIds = new HashSet<>(); + int active = 0; + int terminal = 0; + for (MigrationOperationSnapshot snapshot : snapshots) { + if (!operationIds.add(snapshot.operationId())) { + invalid(); + } + if (snapshot.terminal()) { + terminal++; + } else { + active++; + } + } + int total = snapshots.size(); + if (active > 1 || terminal > FileMigrationOperationStore.HISTORY_LIMIT + || total > FileMigrationOperationStore.HISTORY_LIMIT + 1 + || total == FileMigrationOperationStore.HISTORY_LIMIT + 1 + && (active != 1 || terminal != FileMigrationOperationStore.HISTORY_LIMIT)) { + invalid(); + } + } + + private void invalid() { + throw new IllegalArgumentException("Invalid migration operation collection"); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationFileCodec.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationFileCodec.java new file mode 100644 index 0000000000..dbe655ac13 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationFileCodec.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Strict versioned codec whose vocabulary cannot represent credentials, SQL, or row contents. */ +final class MigrationOperationFileCodec { + + private static final String ABSENT = "-"; + private static final int FIELD_COUNT = 16; + private final MigrationOperationCollectionPolicy collectionPolicy = new MigrationOperationCollectionPolicy(); + + byte[] encode(List snapshots) { + StringBuilder output = new StringBuilder("schema=1\ncount=").append(snapshots.size()).append('\n'); + for (int index = 0; index < snapshots.size(); index++) { + append(output, index, snapshots.get(index)); + } + return output.toString().getBytes(StandardCharsets.UTF_8); + } + + List decode(byte[] encoded) { + Map fields = fields(new String(encoded, StandardCharsets.UTF_8)); + if (!"1".equals(fields.remove("schema"))) { + throw new IllegalArgumentException("Unknown migration operation schema"); + } + int count = integer(fields.remove("count")); + if (count < 0 || count > FileMigrationOperationStore.HISTORY_LIMIT + 1 + || fields.size() != count * FIELD_COUNT) { + throw new IllegalArgumentException("Invalid migration operation count"); + } + List snapshots = new ArrayList<>(count); + for (int index = 0; index < count; index++) { + snapshots.add(snapshot(fields, index)); + } + if (!fields.isEmpty()) { + throw new IllegalArgumentException("Unknown migration operation fields"); + } + collectionPolicy.validate(snapshots); + return snapshots; + } + + private void append(StringBuilder output, int index, MigrationOperationSnapshot value) { + String prefix = index + "."; + field(output, prefix, "operationId", value.operationId()); + field(output, prefix, "state", value.state().name()); + field(output, prefix, "target", value.target().name()); + field(output, prefix, "applyMode", value.applyMode().name()); + field(output, prefix, "stage", value.stage().name()); + field(output, prefix, "progress", Integer.toString(value.progressPercent())); + field(output, prefix, "createdAt", value.createdAt().toString()); + field(output, prefix, "startedAt", optional(value.startedAt())); + field(output, prefix, "completedAt", optional(value.completedAt())); + field(output, prefix, "verification", value.verificationState().name()); + field(output, prefix, "errorCode", optional(value.errorCode())); + field(output, prefix, "rollbackOrigin", optional(value.rollbackOrigin())); + field(output, prefix, "pollMillis", Long.toString(value.nextPollAfterMillis())); + field(output, prefix, "activation", Boolean.toString(value.activationAvailable())); + field(output, prefix, "restart", Boolean.toString(value.restartRequired())); + field(output, prefix, "external", Boolean.toString(value.externalApplyRequired())); + } + + private MigrationOperationSnapshot snapshot(Map fields, int index) { + String prefix = index + "."; + return new MigrationOperationSnapshot( + take(fields, prefix, "operationId"), + value(MigrationOperationState.class, take(fields, prefix, "state")), + value(MigrationTarget.class, take(fields, prefix, "target")), + value(ApplyMode.class, take(fields, prefix, "applyMode")), + value(MigrationStage.class, take(fields, prefix, "stage")), + integer(take(fields, prefix, "progress")), + Instant.parse(take(fields, prefix, "createdAt")), + instant(take(fields, prefix, "startedAt")), + instant(take(fields, prefix, "completedAt")), + value(VerificationState.class, take(fields, prefix, "verification")), + error(take(fields, prefix, "errorCode")), + rollbackOrigin(take(fields, prefix, "rollbackOrigin")), + Long.parseLong(take(fields, prefix, "pollMillis")), + bool(take(fields, prefix, "activation")), + bool(take(fields, prefix, "restart")), + bool(take(fields, prefix, "external"))); + } + + private Map fields(String content) { + Map fields = new HashMap<>(); + for (String line : content.split("\\n", -1)) { + if (line.isEmpty()) { + continue; + } + int separator = line.indexOf('='); + if (separator <= 0 || separator == line.length() - 1 + || fields.put(line.substring(0, separator), line.substring(separator + 1)) != null) { + throw new IllegalArgumentException("Malformed migration operation field"); + } + } + return fields; + } + + private String take(Map fields, String prefix, String name) { + String value = fields.remove(prefix + name); + if (value == null) { + throw new IllegalArgumentException("Missing migration operation field"); + } + return value; + } + + private void field(StringBuilder output, String prefix, String name, String value) { + if (value.indexOf('\n') >= 0 || value.indexOf('=') >= 0) { + throw new IllegalArgumentException("Unsafe migration operation field"); + } + output.append(prefix).append(name).append('=').append(value).append('\n'); + } + + private String optional(Object value) { + return value == null ? ABSENT : value.toString(); + } + + private Instant instant(String value) { + return ABSENT.equals(value) ? null : Instant.parse(value); + } + + private SetupErrorCode error(String value) { + return ABSENT.equals(value) ? null : value(SetupErrorCode.class, value); + } + + private MigrationRollbackOrigin rollbackOrigin(String value) { + return ABSENT.equals(value) ? null : value(MigrationRollbackOrigin.class, value); + } + + private int integer(String value) { + return Integer.parseInt(value); + } + + private boolean bool(String value) { + if (!"true".equals(value) && !"false".equals(value)) { + throw new IllegalArgumentException("Invalid migration operation boolean"); + } + return Boolean.parseBoolean(value); + } + + private > T value(Class type, String value) { + return Enum.valueOf(type, value); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationFilePublisher.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationFilePublisher.java new file mode 100644 index 0000000000..11be45c425 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationFilePublisher.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.UUID; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; + +/** Atomic publication seam kept separate from operation lifecycle and encoding. */ +final class MigrationOperationFilePublisher implements FileMigrationOperationStore.Publisher { + + private final Path installationRoot; + + MigrationOperationFilePublisher(Path installationRoot) { + this.installationRoot = installationRoot; + } + + @Override + public void publish(Path target, byte[] content) throws IOException { + Path temporary = target.getParent().resolve(".migration-operations-" + UUID.randomUUID() + ".tmp"); + try { + SecureSetupFile.create(installationRoot, temporary, content); + SecureSetupFile.atomicReplace(installationRoot, temporary, target); + } finally { + Files.deleteIfExists(temporary); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationSnapshot.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationSnapshot.java new file mode 100644 index 0000000000..0b295a1673 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationSnapshot.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.time.Instant; +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationView; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Secret-free durable state for one H2 metadata migration operation. */ +public record MigrationOperationSnapshot( + String operationId, + MigrationOperationState state, + MigrationTarget target, + ApplyMode applyMode, + MigrationStage stage, + int progressPercent, + Instant createdAt, + Instant startedAt, + Instant completedAt, + VerificationState verificationState, + SetupErrorCode errorCode, + MigrationRollbackOrigin rollbackOrigin, + long nextPollAfterMillis, + boolean activationAvailable, + boolean restartRequired, + boolean externalApplyRequired) { + + public MigrationOperationSnapshot { + Objects.requireNonNull(applyMode, "applyMode"); + new MigrationView(operationId, state, MetadataDatabaseKind.H2, target, stage, progressPercent, + createdAt, startedAt, completedAt, verificationState, errorCode, nextPollAfterMillis, + activationAvailable, restartRequired, externalApplyRequired); + validateRollback(state, stage, verificationState, errorCode, rollbackOrigin); + } + + public boolean terminal() { + return state == MigrationOperationState.SUCCEEDED + || state == MigrationOperationState.FAILED + || state == MigrationOperationState.ROLLED_BACK; + } + + private static void validateRollback( + MigrationOperationState state, MigrationStage stage, VerificationState verification, + SetupErrorCode errorCode, MigrationRollbackOrigin origin) { + boolean rollingBack = state == MigrationOperationState.RUNNING && stage == MigrationStage.ROLLING_BACK; + boolean rolledBack = state == MigrationOperationState.ROLLED_BACK; + if ((origin != null) != (rollingBack || rolledBack)) { + throw new IllegalArgumentException("Migration rollback origin is inconsistent"); + } + if (origin != null && (verification != origin.verificationState() + || rolledBack && errorCode != origin.errorCode())) { + throw new IllegalArgumentException("Migration rollback origin is inconsistent"); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationStore.java new file mode 100644 index 0000000000..487cdbb4e6 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationStore.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.List; +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; + +/** Persistence port with optimistic expected-state transitions. */ +public interface MigrationOperationStore { + + MigrationOperationSnapshot create(MigrationOperationSnapshot snapshot); + + Optional find(String operationId); + + List history(); + + MigrationOperationSnapshot compareAndTransition( + String operationId, MigrationOperationState expectedState, MigrationOperationSnapshot replacement); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationStoreException.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationStoreException.java new file mode 100644 index 0000000000..bbbe459d64 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationStoreException.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Stable store failure that never retains provider messages, paths, or operation payloads. */ +public final class MigrationOperationStoreException extends RuntimeException { + + private final SetupErrorCode errorCode; + + MigrationOperationStoreException(SetupErrorCode errorCode) { + super("Migration operation store failed: " + Objects.requireNonNull(errorCode, "errorCode").value()); + this.errorCode = errorCode; + } + + public SetupErrorCode errorCode() { + return errorCode; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicy.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicy.java new file mode 100644 index 0000000000..d2892c6fa3 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicy.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Monotonic lifecycle boundary layered on the frozen migration projection validator. */ +final class MigrationOperationTransitionPolicy { + + void requireAllowed(MigrationOperationSnapshot current, MigrationOperationSnapshot next) { + if (!current.operationId().equals(next.operationId()) + || current.target() != next.target() + || current.applyMode() != next.applyMode() + || !current.createdAt().equals(next.createdAt()) + || current.startedAt() != null && !current.startedAt().equals(next.startedAt()) + || next.progressPercent() < current.progressPercent() + || !allowedEdge(current, next)) { + throw new MigrationOperationStoreException(SetupErrorCode.OPERATION_CONFLICT); + } + } + + private boolean allowedEdge(MigrationOperationSnapshot current, MigrationOperationSnapshot next) { + return switch (current.state()) { + case PENDING -> pendingExit(next); + case RUNNING -> runningExit(current, next); + case READY_TO_ACTIVATE -> readyExit(next); + case AWAITING_EXTERNAL_APPLY -> externalExit(next); + case AWAITING_RESTART -> restartExit(next); + case SUCCEEDED, FAILED, ROLLED_BACK -> false; + }; + } + + private boolean pendingExit(MigrationOperationSnapshot next) { + return runningAt(next, MigrationStage.COPYING, VerificationState.PENDING) + || failedWith(next, SetupErrorCode.MIGRATION_COPY_FAILED); + } + + private boolean runningExit(MigrationOperationSnapshot current, MigrationOperationSnapshot next) { + return switch (current.stage()) { + case COPYING -> runningAt(next, MigrationStage.COPYING, VerificationState.PENDING) + || runningAt(next, MigrationStage.VERIFYING, VerificationState.RUNNING) + || failedWith(next, SetupErrorCode.MIGRATION_COPY_FAILED); + case VERIFYING -> verifiedExit(current, next) + || failedWith(next, SetupErrorCode.MIGRATION_VERIFICATION_FAILED); + case ACTIVATING -> runningAt(next, MigrationStage.ACTIVATING, VerificationState.SUCCEEDED) + || rollingBackAt(next, MigrationRollbackOrigin.ACTIVATION_FAILURE) + || next.state() == MigrationOperationState.AWAITING_RESTART + || next.state() == MigrationOperationState.SUCCEEDED + || failedWith(next, SetupErrorCode.MIGRATION_ACTIVATION_FAILED); + case ROLLING_BACK -> rollbackContinues(current, next) || rollbackCompletes(current, next); + default -> false; + }; + } + + private boolean readyExit(MigrationOperationSnapshot next) { + return runningAt(next, MigrationStage.ACTIVATING, VerificationState.SUCCEEDED) + || rollingBackAt(next, MigrationRollbackOrigin.ACTIVATION_FAILURE); + } + + private boolean externalExit(MigrationOperationSnapshot next) { + return next.state() == MigrationOperationState.AWAITING_RESTART + || next.state() == MigrationOperationState.SUCCEEDED + || failedWith(next, SetupErrorCode.MIGRATION_ACTIVATION_FAILED); + } + + private boolean restartExit(MigrationOperationSnapshot next) { + return next.state() == MigrationOperationState.SUCCEEDED + || failedWith(next, SetupErrorCode.RESTART_FAILED) + || rollingBackAt(next, MigrationRollbackOrigin.RESTART_FAILURE); + } + + private boolean verifiedExit(MigrationOperationSnapshot current, MigrationOperationSnapshot next) { + return current.applyMode() == ApplyMode.MANAGED_WRITE + ? next.state() == MigrationOperationState.READY_TO_ACTIVATE + : next.state() == MigrationOperationState.AWAITING_EXTERNAL_APPLY; + } + + private boolean rollingBackAt(MigrationOperationSnapshot next, MigrationRollbackOrigin origin) { + return runningAt(next, MigrationStage.ROLLING_BACK, origin.verificationState()) + && next.rollbackOrigin() == origin; + } + + private boolean rollbackContinues(MigrationOperationSnapshot current, MigrationOperationSnapshot next) { + return rollingBackAt(next, current.rollbackOrigin()) + && next.verificationState() == current.verificationState(); + } + + private boolean rollbackCompletes(MigrationOperationSnapshot current, MigrationOperationSnapshot next) { + return next.state() == MigrationOperationState.ROLLED_BACK + && next.rollbackOrigin() == current.rollbackOrigin() + && next.verificationState() == current.verificationState() + && next.errorCode() == current.rollbackOrigin().errorCode(); + } + + private boolean runningAt( + MigrationOperationSnapshot next, MigrationStage stage, VerificationState verification) { + return next.state() == MigrationOperationState.RUNNING + && next.stage() == stage && next.verificationState() == verification; + } + + private boolean failedWith(MigrationOperationSnapshot next, SetupErrorCode errorCode) { + return next.state() == MigrationOperationState.FAILED && next.errorCode() == errorCode; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationRollbackOrigin.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationRollbackOrigin.java new file mode 100644 index 0000000000..8ef797bcc0 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationRollbackOrigin.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Frozen cause of a rollback, independent of mutable stage and terminal projection fields. */ +public enum MigrationRollbackOrigin { + VERIFICATION_FAILURE(VerificationState.FAILED, SetupErrorCode.MIGRATION_VERIFICATION_FAILED), + ACTIVATION_FAILURE(VerificationState.SUCCEEDED, SetupErrorCode.MIGRATION_ACTIVATION_FAILED), + RESTART_FAILURE(VerificationState.SUCCEEDED, SetupErrorCode.RESTART_FAILED); + + private final VerificationState verificationState; + private final SetupErrorCode errorCode; + + MigrationRollbackOrigin(VerificationState verificationState, SetupErrorCode errorCode) { + this.verificationState = verificationState; + this.errorCode = errorCode; + } + + VerificationState verificationState() { + return verificationState; + } + + SetupErrorCode errorCode() { + return errorCode; + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileLockProcessMain.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileLockProcessMain.java new file mode 100644 index 0000000000..f82b8d8a37 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileLockProcessMain.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.security; + +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.time.Duration; +import java.time.Instant; +import java.util.Set; + +/** Child-JVM fixture proving the cooperative lock's operating-system boundary. */ +public final class SecureSetupFileLockProcessMain { + + private static final String LOCK_PATH = "data/config/.cooperative-test.lock"; + + private SecureSetupFileLockProcessMain() { + } + + public static void main(String[] arguments) throws Exception { + Path root = Path.of(arguments[0]); + Path ready = Path.of(arguments[1]); + Path entered = Path.of(arguments[2]); + Path release = Path.of(arguments[3]); + SecureSetupFileLock lock = new SecureSetupFileLock(root, LOCK_PATH); + Path canonicalRoot = root.toRealPath(); + Path lockFile = canonicalRoot.resolve(LOCK_PATH); + if (!SecureSetupFile.existsInsideRootWithoutLinks(canonicalRoot, lockFile) + || !SecureSetupFile.isOwnerOnlyRegularFile(lockFile)) { + throw new IllegalStateException("Child lock probe path is invalid"); + } + requireOperatingSystemConflict(lockFile); + Files.writeString(ready, "blocked-by-os-lock"); + lock.execute(() -> { + Files.createFile(entered); + awaitRelease(release); + }); + } + + private static void requireOperatingSystemConflict(Path lockFile) throws Exception { + try (FileChannel channel = FileChannel.open(lockFile, + Set.of(StandardOpenOption.READ, StandardOpenOption.WRITE, LinkOption.NOFOLLOW_LINKS))) { + try (FileLock acquired = channel.tryLock()) { + if (acquired != null) { + throw new IllegalStateException("Child unexpectedly acquired the parent OS lock"); + } + } + } + } + + private static void awaitRelease(Path release) { + Instant deadline = Instant.now().plus(Duration.ofSeconds(10)); + while (!Files.exists(release) && Instant.now().isBefore(deadline)) { + Thread.onSpinWait(); + } + if (!Files.exists(release)) { + throw new IllegalStateException("Parent did not release child lock fixture"); + } + } + +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileLockTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileLockTest.java new file mode 100644 index 0000000000..eb4bfaeafb --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileLockTest.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.security; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.locks.LockSupport; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class SecureSetupFileLockTest { + + private static final String LOCK_PATH = "data/config/.cooperative-test.lock"; + + @TempDir + private Path root; + + @Test + void canonicalAliasesShareTheSameJvmLock() throws Exception { + Path alias = root.resolve("alias"); + Files.createSymbolicLink(alias, root); + SecureSetupFileLock lexical = new SecureSetupFileLock(root, LOCK_PATH); + SecureSetupFileLock canonical = new SecureSetupFileLock(alias, LOCK_PATH); + assertSerialized(lexical, canonical, null); + } + + @Test + void replacementInodeCannotSplitCooperatingJvmContexts() throws Exception { + Path alias = root.resolve("replacement-alias"); + Files.createSymbolicLink(alias, root); + SecureSetupFileLock lexical = new SecureSetupFileLock(root, LOCK_PATH); + SecureSetupFileLock canonical = new SecureSetupFileLock(alias, LOCK_PATH); + Path lockFile = root.resolve(LOCK_PATH); + assertSerialized(lexical, canonical, () -> { + Files.delete(lockFile); + SecureSetupFile.create(root, lockFile, "replacement\n".getBytes(StandardCharsets.UTF_8)); + }); + } + + @Test + void createsMissingTrustedRootWithoutWeakeningFileBoundary() throws Exception { + Path missingRoot = root.resolve("missing-installation-root"); + + new SecureSetupFileLock(missingRoot, LOCK_PATH).execute(() -> { }); + + assertThat(missingRoot).isDirectory(); + assertThat(SecureSetupFile.isOwnerOnlyRegularFile(missingRoot.resolve(LOCK_PATH))).isTrue(); + } + + @Test + void childProcessLockIsSerialized() throws Exception { + Path ready = root.resolve("child-ready"); + Path entered = root.resolve("child-entered"); + Path release = root.resolve("child-release"); + String java = Path.of(System.getProperty("java.home"), "bin", "java").toString(); + ProcessBuilder command = new ProcessBuilder(java, "-cp", System.getProperty("java.class.path"), + SecureSetupFileLockProcessMain.class.getName(), root.toString(), ready.toString(), + entered.toString(), release.toString()) + .redirectErrorStream(true); + Process[] child = new Process[1]; + try { + SecureSetupFileLock parent = new SecureSetupFileLock(root, LOCK_PATH); + parent.execute(() -> { + child[0] = command.start(); + assertThat(awaitFileContent(ready, child[0])).isEqualTo("blocked-by-os-lock"); + if (Files.exists(entered)) { + throw new IOException("Child entered the parent lock critical section"); + } + }); + awaitFile(entered); + Files.createFile(release); + assertThat(child[0].waitFor(5, TimeUnit.SECONDS)).isTrue(); + assertThat(child[0].exitValue()).isZero(); + } finally { + if (child[0] != null) { + child[0].destroyForcibly(); + } + } + } + + private void assertSerialized( + SecureSetupFileLock first, SecureSetupFileLock second, CheckedAction whileLocked) throws Exception { + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + try (ExecutorService executor = Executors.newFixedThreadPool(2)) { + Future owner = executor.submit(() -> first.execute(() -> { + entered.countDown(); + awaitLatch(release); + return true; + })); + assertThat(entered.await(5, TimeUnit.SECONDS)).isTrue(); + if (whileLocked != null) { + whileLocked.run(); + } + Future waiter = executor.submit(() -> second.execute(() -> true)); + assertBlocked(waiter); + release.countDown(); + if (whileLocked == null) { + assertThat(owner.get(5, TimeUnit.SECONDS)).isTrue(); + assertThat(waiter.get(5, TimeUnit.SECONDS)).isTrue(); + } else { + assertThatThrownBy(() -> owner.get(5, TimeUnit.SECONDS)) + .hasRootCauseInstanceOf(IOException.class); + assertThatThrownBy(() -> waiter.get(5, TimeUnit.SECONDS)) + .hasRootCauseInstanceOf(IOException.class); + } + } + } + + private void assertBlocked(Future acquisition) { + try { + acquisition.get(200, TimeUnit.MILLISECONDS); + throw new AssertionError("Lock acquisition completed inside another critical section"); + } catch (TimeoutException expected) { + // Expected proof that another context still owns the critical section. + } catch (ExecutionException failure) { + throw new AssertionError("Lock acquisition failed instead of waiting", failure.getCause()); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new AssertionError("Interrupted while proving lock serialization", interrupted); + } + } + + private void awaitFile(Path file) throws IOException { + Instant deadline = Instant.now().plus(Duration.ofSeconds(5)); + while (!Files.exists(file) && Instant.now().isBefore(deadline)) { + LockSupport.parkNanos(Duration.ofMillis(10).toNanos()); + } + if (!Files.exists(file)) { + throw new IOException("Timed out waiting for child lock fixture"); + } + } + + private String awaitFileContent(Path file, Process child) throws IOException { + Instant deadline = Instant.now().plus(Duration.ofSeconds(5)); + while (Instant.now().isBefore(deadline)) { + if (Files.exists(file) && Files.size(file) > 0) { + return Files.readString(file); + } + Thread.onSpinWait(); + } + if (!child.isAlive()) { + throw new IOException("Child lock probe exited: " + + new String(child.getInputStream().readAllBytes(), StandardCharsets.UTF_8)); + } + throw new IOException("Timed out waiting for child lock evidence"); + } + + private void awaitLatch(CountDownLatch latch) throws IOException { + try { + if (!latch.await(5, TimeUnit.SECONDS)) { + throw new IOException("Timed out waiting for lock test release"); + } + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while waiting for lock test release", interrupted); + } + } + + @FunctionalInterface + private interface CheckedAction { + void run() throws Exception; + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileTest.java index def5c980cd..3429e82dc3 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileTest.java @@ -24,6 +24,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assumptions.assumeFalse; import static org.junit.jupiter.api.Assumptions.assumeTrue; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.FileStore; import java.nio.file.Files; @@ -78,7 +79,7 @@ class SecureSetupFileTest { Path linkedDirectory = temporaryDirectory.resolve("linked"); Files.createSymbolicLink(linkedDirectory, realDirectory); - assertThrows(java.io.IOException.class, () -> SecureSetupFile.create( + assertThrows(IOException.class, () -> SecureSetupFile.create( temporaryDirectory, linkedDirectory.resolve("nested").resolve("secret"), "secret-content".getBytes(StandardCharsets.UTF_8))); @@ -92,9 +93,23 @@ class SecureSetupFileTest { Path linkedDirectory = trustedRoot.resolve("linked"); Files.createSymbolicLink(linkedDirectory, outside); - assertThrows(java.io.IOException.class, () -> SecureSetupFile.create( + assertThrows(IOException.class, () -> SecureSetupFile.create( trustedRoot, linkedDirectory.resolve("nested").resolve("secret"), "secret-content".getBytes(StandardCharsets.UTF_8))); } + + @Test + void reportsCommittedReplacementWhenParentSyncFails() throws Exception { + Path target = temporaryDirectory.resolve("state"); + Path replacement = temporaryDirectory.resolve("replacement"); + SecureSetupFile.create(temporaryDirectory, target, "old".getBytes(StandardCharsets.UTF_8)); + SecureSetupFile.create(temporaryDirectory, replacement, "new".getBytes(StandardCharsets.UTF_8)); + + assertThrows(CommittedSetupFileDurabilityException.class, () -> SecureSetupFile.atomicReplace( + temporaryDirectory, replacement, target, ignored -> { + throw new IOException("simulated parent fsync failure"); + })); + assertArrayEquals("new".getBytes(StandardCharsets.UTF_8), Files.readAllBytes(target)); + } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStoreTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStoreTest.java new file mode 100644 index 0000000000..3b399eee7d --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStoreTest.java @@ -0,0 +1,328 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.security.CommittedSetupFileDurabilityException; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class FileMigrationOperationStoreTest { + + @TempDir + private Path root; + + @Test + void createsAdvancesAndRecoversWithoutSecrets() throws Exception { + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + MigrationOperationSnapshot pending = pending("migration-1", Instant.parse("2026-08-09T01:00:00Z")); + assertThat(store.create(pending)).isEqualTo(pending); + + MigrationOperationSnapshot running = running(pending, 25); + assertThat(store.compareAndTransition(pending.operationId(), MigrationOperationState.PENDING, running)) + .isEqualTo(running); + assertThat(new FileMigrationOperationStore(root).find(pending.operationId())).contains(running); + assertThat(SecureSetupFile.isOwnerOnlyRegularFile(root.resolve(FileMigrationOperationStore.RELATIVE_PATH))) + .isTrue(); + + String persisted = Files.readString(root.resolve(FileMigrationOperationStore.RELATIVE_PATH)); + assertThat(persisted).doesNotContain("jdbc:", "username", "password", "SELECT", "secret-value"); + } + + @Test + void rejectsStaleAndNonMonotonicTransitions() throws Exception { + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + MigrationOperationSnapshot pending = pending("migration-1", Instant.parse("2026-08-09T01:00:00Z")); + store.create(pending); + MigrationOperationSnapshot running = running(pending, 25); + store.compareAndTransition(pending.operationId(), MigrationOperationState.PENDING, running); + + assertStoreError(SetupErrorCode.OPERATION_CONFLICT, () -> store.compareAndTransition( + pending.operationId(), MigrationOperationState.PENDING, running(pending, 50))); + assertStoreError(SetupErrorCode.OPERATION_CONFLICT, () -> store.compareAndTransition( + pending.operationId(), MigrationOperationState.RUNNING, running(pending, 10))); + } + + @Test + void permitsOnlyOneActiveOperationAndTrimsTerminalHistory() throws Exception { + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + MigrationOperationSnapshot active = pending("active", Instant.parse("2026-08-09T01:00:00Z")); + store.create(active); + assertStoreError(SetupErrorCode.OPERATION_CONFLICT, + () -> store.create(pending("other", active.createdAt().plusSeconds(1)))); + complete(store, active); + + for (int index = 0; index < FileMigrationOperationStore.HISTORY_LIMIT + 3; index++) { + MigrationOperationSnapshot item = pending("history-" + index, active.createdAt().plusSeconds(index + 2)); + store.create(item); + complete(store, item); + } + assertThat(store.history()).hasSize(FileMigrationOperationStore.HISTORY_LIMIT); + assertThat(store.find("active")).isEmpty(); + } + + @Test + void failsClosedForCorruptionUnknownVersionAndUnsafeFiles() throws Exception { + Path file = root.resolve(FileMigrationOperationStore.RELATIVE_PATH); + Files.createDirectories(file.getParent()); + Files.writeString(file, "schema=99\n", StandardCharsets.UTF_8); + ownerOnly(file); + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> new FileMigrationOperationStore(root).history()); + + Files.writeString(file, "schema=1\ncount=1\n", StandardCharsets.UTF_8); + ownerOnly(file); + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> new FileMigrationOperationStore(root).history()); + + Files.delete(file); + Path outside = Files.createTempFile("migration-operations", ".outside"); + Files.createSymbolicLink(file, outside); + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> new FileMigrationOperationStore(root).history()); + } + + @Test + void rejectsNonOwnerOnlyFileAndSymlinkedConfigurationDirectory() throws Exception { + Path file = root.resolve(FileMigrationOperationStore.RELATIVE_PATH); + Files.createDirectories(file.getParent()); + Files.writeString(file, "schema=1\ncount=0\n", StandardCharsets.UTF_8); + if (Files.getFileStore(file).supportsFileAttributeView("posix")) { + Files.setPosixFilePermissions(file, PosixFilePermissions.fromString("rw-r--r--")); + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> new FileMigrationOperationStore(root).history()); + } + + Path symlinkRoot = root.resolve("symlink-root"); + Path symlinkConfig = symlinkRoot.resolve("data/config"); + Files.createDirectories(symlinkConfig.getParent()); + Path outsideDirectory = Files.createTempDirectory("migration-config-outside"); + Files.createSymbolicLink(symlinkConfig, outsideDirectory); + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> new FileMigrationOperationStore(symlinkRoot).history()); + } + + @Test + void rejectsPersistedCollectionWithDuplicateOperationIds() throws Exception { + MigrationOperationSnapshot first = pending("duplicate", Instant.parse("2026-08-09T01:00:00Z")); + writePersisted(List.of(succeeded(first), succeeded(first))); + + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> new FileMigrationOperationStore(root).history()); + } + + @Test + void rejectsPersistedCollectionWithTwoActiveOperations() throws Exception { + Instant created = Instant.parse("2026-08-09T01:00:00Z"); + writePersisted(List.of(pending("first", created), pending("second", created.plusSeconds(1)))); + + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> new FileMigrationOperationStore(root).history()); + } + + @Test + void rejectsPersistedCollectionBeyondTerminalHistoryLimit() throws Exception { + Instant created = Instant.parse("2026-08-09T01:00:00Z"); + List terminal = new ArrayList<>(); + for (int index = 0; index < FileMigrationOperationStore.HISTORY_LIMIT + 1; index++) { + terminal.add(succeeded(pending("terminal-" + index, created.plusSeconds(index)))); + } + writePersisted(terminal); + + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> new FileMigrationOperationStore(root).history()); + } + + @Test + void rollbackOriginRoundTripsAndCrossSourceCorruptionRequiresRecovery() throws Exception { + MigrationOperationSnapshot rolledBack = rolledBack( + pending("rolled-back", Instant.parse("2026-08-09T01:00:00Z")), + MigrationRollbackOrigin.ACTIVATION_FAILURE); + writePersisted(List.of(rolledBack)); + assertThat(new FileMigrationOperationStore(root).history()).containsExactly(rolledBack); + + Path file = root.resolve(FileMigrationOperationStore.RELATIVE_PATH); + String corrupted = Files.readString(file).replace( + "rollbackOrigin=ACTIVATION_FAILURE", "rollbackOrigin=RESTART_FAILURE"); + Files.writeString(file, corrupted, StandardCharsets.UTF_8); + ownerOnly(file); + + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> new FileMigrationOperationStore(root).history()); + } + + @Test + void previousFieldSetWithoutRollbackOriginRequiresRecovery() throws Exception { + MigrationOperationSnapshot pending = pending("legacy-fields", Instant.parse("2026-08-09T01:00:00Z")); + String previousFieldSet = new String( + new MigrationOperationFileCodec().encode(List.of(pending)), StandardCharsets.UTF_8) + .replace("0.rollbackOrigin=-\n", ""); + Path file = root.resolve(FileMigrationOperationStore.RELATIVE_PATH); + SecureSetupFile.create(root, file, previousFieldSet.getBytes(StandardCharsets.UTF_8)); + + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> new FileMigrationOperationStore(root).history()); + } + + @Test + void failedAtomicPublicationPreservesPreviousState() throws Exception { + FileMigrationOperationStore initial = new FileMigrationOperationStore(root); + MigrationOperationSnapshot pending = pending("migration-1", Instant.parse("2026-08-09T01:00:00Z")); + initial.create(pending); + FileMigrationOperationStore failing = new FileMigrationOperationStore(root, (target, content) -> { + throw new AtomicMoveNotSupportedException( + "provider path jdbc:secret", "password=secret-value", "SELECT private"); + }); + + assertStoreError(SetupErrorCode.CONFIG_WRITE_FAILED, () -> failing.compareAndTransition( + pending.operationId(), MigrationOperationState.PENDING, running(pending, 10))); + assertThat(new FileMigrationOperationStore(root).find(pending.operationId())).contains(pending); + } + + @Test + void committedPublicationWithUncertainDirectoryDurabilityRequiresRecovery() throws Exception { + FileMigrationOperationStore initial = new FileMigrationOperationStore(root); + MigrationOperationSnapshot pending = pending("migration-1", Instant.parse("2026-08-09T01:00:00Z")); + initial.create(pending); + MigrationOperationSnapshot running = running(pending, 10); + MigrationOperationFilePublisher committed = new MigrationOperationFilePublisher(root); + FileMigrationOperationStore uncertain = new FileMigrationOperationStore(root, (target, content) -> { + committed.publish(target, content); + throw new CommittedSetupFileDurabilityException(); + }); + + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, () -> uncertain.compareAndTransition( + pending.operationId(), MigrationOperationState.PENDING, running)); + assertThat(new FileMigrationOperationStore(root).find(pending.operationId())).contains(running); + } + + @Test + void twoInstancesSerializeCompareAndTransition() throws Exception { + FileMigrationOperationStore first = new FileMigrationOperationStore(root); + FileMigrationOperationStore second = new FileMigrationOperationStore(root); + MigrationOperationSnapshot pending = pending("migration-1", Instant.parse("2026-08-09T01:00:00Z")); + first.create(pending); + List> attempts = List.of( + () -> transition(first, pending, 20), () -> transition(second, pending, 30)); + List> results; + try (ExecutorService executor = Executors.newFixedThreadPool(2)) { + results = new ArrayList<>(executor.invokeAll(attempts)); + } + assertThat(results.get(0).get() ^ results.get(1).get()).isTrue(); + assertThat(new FileMigrationOperationStore(root).find(pending.operationId()).orElseThrow().progressPercent()) + .isIn(20, 30); + } + + private boolean transition(FileMigrationOperationStore store, MigrationOperationSnapshot pending, int progress) { + try { + store.compareAndTransition(pending.operationId(), MigrationOperationState.PENDING, + running(pending, progress)); + return true; + } catch (MigrationOperationStoreException conflict) { + assertThat(conflict.errorCode()).isEqualTo(SetupErrorCode.OPERATION_CONFLICT); + return false; + } + } + + private static MigrationOperationSnapshot pending(String id, Instant createdAt) { + return new MigrationOperationSnapshot(id, MigrationOperationState.PENDING, MigrationTarget.MYSQL, + ApplyMode.MANAGED_WRITE, MigrationStage.QUEUED, 0, createdAt, null, null, + VerificationState.PENDING, null, null, 1000, false, false, false); + } + + private static MigrationOperationSnapshot running(MigrationOperationSnapshot pending, int progress) { + return new MigrationOperationSnapshot(pending.operationId(), MigrationOperationState.RUNNING, pending.target(), + pending.applyMode(), MigrationStage.COPYING, progress, pending.createdAt(), + pending.createdAt().plusSeconds(1), null, VerificationState.PENDING, null, null, 1000, + false, false, false); + } + + private static MigrationOperationSnapshot succeeded(MigrationOperationSnapshot pending) { + Instant started = pending.createdAt().plusSeconds(1); + return new MigrationOperationSnapshot(pending.operationId(), MigrationOperationState.SUCCEEDED, pending.target(), + pending.applyMode(), MigrationStage.COMPLETED, 100, pending.createdAt(), started, + started.plusSeconds(1), VerificationState.SUCCEEDED, null, null, 0, false, false, false); + } + + private static MigrationOperationSnapshot rolledBack( + MigrationOperationSnapshot pending, MigrationRollbackOrigin origin) { + Instant started = pending.createdAt().plusSeconds(1); + return new MigrationOperationSnapshot(pending.operationId(), MigrationOperationState.ROLLED_BACK, + pending.target(), pending.applyMode(), MigrationStage.ROLLED_BACK, 100, + pending.createdAt(), started, started.plusSeconds(1), origin.verificationState(), + origin.errorCode(), origin, 0, false, false, false); + } + + private static void complete(FileMigrationOperationStore store, MigrationOperationSnapshot pending) { + MigrationOperationSnapshot running = running(pending, 25); + store.compareAndTransition(pending.operationId(), MigrationOperationState.PENDING, running); + MigrationOperationSnapshot verifying = new MigrationOperationSnapshot( + pending.operationId(), MigrationOperationState.RUNNING, pending.target(), pending.applyMode(), + MigrationStage.VERIFYING, 100, pending.createdAt(), pending.createdAt().plusSeconds(1), null, + VerificationState.RUNNING, null, null, 1000, false, false, false); + store.compareAndTransition(pending.operationId(), MigrationOperationState.RUNNING, verifying); + MigrationOperationSnapshot ready = new MigrationOperationSnapshot( + pending.operationId(), MigrationOperationState.READY_TO_ACTIVATE, pending.target(), pending.applyMode(), + MigrationStage.READY_TO_ACTIVATE, 100, pending.createdAt(), pending.createdAt().plusSeconds(1), null, + VerificationState.SUCCEEDED, null, null, 0, true, false, false); + store.compareAndTransition(pending.operationId(), MigrationOperationState.RUNNING, ready); + MigrationOperationSnapshot activating = new MigrationOperationSnapshot( + pending.operationId(), MigrationOperationState.RUNNING, pending.target(), pending.applyMode(), + MigrationStage.ACTIVATING, 100, pending.createdAt(), pending.createdAt().plusSeconds(1), null, + VerificationState.SUCCEEDED, null, null, 1000, false, false, false); + store.compareAndTransition(pending.operationId(), MigrationOperationState.READY_TO_ACTIVATE, activating); + store.compareAndTransition(pending.operationId(), MigrationOperationState.RUNNING, succeeded(pending)); + } + + private static void assertStoreError(SetupErrorCode expected, ThrowingAction action) { + assertThatThrownBy(action::run).isInstanceOfSatisfying(MigrationOperationStoreException.class, + failure -> assertThat(failure.errorCode()).isEqualTo(expected)) + .hasMessageNotContaining("jdbc") + .hasMessageNotContaining("password") + .hasMessageNotContaining("SELECT") + .hasMessageNotContaining("/"); + } + + private static void ownerOnly(Path file) throws IOException { + if (Files.getFileStore(file).supportsFileAttributeView("posix")) { + Files.setPosixFilePermissions(file, PosixFilePermissions.fromString("rw-------")); + } + } + + private void writePersisted(List snapshots) throws IOException { + byte[] encoded = new MigrationOperationFileCodec().encode(snapshots); + SecureSetupFile.create(root, root.resolve(FileMigrationOperationStore.RELATIVE_PATH), encoded); + } + + @FunctionalInterface + private interface ThrowingAction { + void run() throws Exception; + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicyTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicyTest.java new file mode 100644 index 0000000000..07db9e74a3 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicyTest.java @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.time.Instant; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.junit.jupiter.api.Test; + +class MigrationOperationTransitionPolicyTest { + + private static final Instant CREATED = Instant.parse("2026-08-09T01:00:00Z"); + private static final Instant STARTED = CREATED.plusSeconds(1); + private static final Instant COMPLETED = STARTED.plusSeconds(1); + private final MigrationOperationTransitionPolicy policy = new MigrationOperationTransitionPolicy(); + + @Test + void acceptsExplicitManagedLifecycleEdges() { + MigrationOperationSnapshot pending = snapshot(MigrationOperationState.PENDING, MigrationStage.QUEUED, + 0, null, null, VerificationState.PENDING, null, 1000, false, false, false); + MigrationOperationSnapshot copying = snapshot(MigrationOperationState.RUNNING, MigrationStage.COPYING, + 25, STARTED, null, VerificationState.PENDING, null, 1000, false, false, false); + MigrationOperationSnapshot verifying = snapshot(MigrationOperationState.RUNNING, MigrationStage.VERIFYING, + 100, STARTED, null, VerificationState.RUNNING, null, 1000, false, false, false); + MigrationOperationSnapshot ready = snapshot(MigrationOperationState.READY_TO_ACTIVATE, + MigrationStage.READY_TO_ACTIVATE, 100, STARTED, null, VerificationState.SUCCEEDED, + null, 0, true, false, false); + MigrationOperationSnapshot activating = snapshot(MigrationOperationState.RUNNING, MigrationStage.ACTIVATING, + 100, STARTED, null, VerificationState.SUCCEEDED, null, 1000, false, false, false); + MigrationOperationSnapshot restart = snapshot(MigrationOperationState.AWAITING_RESTART, + MigrationStage.AWAITING_RESTART, 100, STARTED, null, VerificationState.SUCCEEDED, + null, 1000, false, true, false); + MigrationOperationSnapshot succeeded = snapshot(MigrationOperationState.SUCCEEDED, MigrationStage.COMPLETED, + 100, STARTED, COMPLETED, VerificationState.SUCCEEDED, null, 0, false, false, false); + + assertAllowed(pending, copying); + assertAllowed(copying, verifying); + assertAllowed(verifying, ready); + assertAllowed(ready, activating); + assertAllowed(activating, restart); + assertAllowed(restart, succeeded); + } + + @Test + void acceptsExternalFailedAndRolledBackExitsButTerminalStatesStayClosed() { + MigrationOperationSnapshot verifying = external(MigrationOperationState.RUNNING, MigrationStage.VERIFYING, + VerificationState.RUNNING, null, false, null); + MigrationOperationSnapshot external = external(MigrationOperationState.AWAITING_EXTERNAL_APPLY, + MigrationStage.AWAITING_EXTERNAL_APPLY, VerificationState.SUCCEEDED, null, true, null); + MigrationOperationSnapshot failed = snapshot(MigrationOperationState.FAILED, MigrationStage.FAILED, + 25, STARTED, COMPLETED, VerificationState.PENDING, SetupErrorCode.MIGRATION_COPY_FAILED, + 0, false, false, false); + MigrationOperationSnapshot rollingBack = rollingBack(MigrationRollbackOrigin.ACTIVATION_FAILURE); + MigrationOperationSnapshot rolledBack = rolledBack(MigrationRollbackOrigin.ACTIVATION_FAILURE, + SetupErrorCode.MIGRATION_ACTIVATION_FAILED); + + assertAllowed(verifying, external); + assertAllowed(snapshot(MigrationOperationState.RUNNING, MigrationStage.COPYING, 25, STARTED, null, + VerificationState.PENDING, null, 1000, false, false, false), failed); + assertAllowed(snapshot(MigrationOperationState.READY_TO_ACTIVATE, MigrationStage.READY_TO_ACTIVATE, + 100, STARTED, null, VerificationState.SUCCEEDED, null, 0, true, false, false), rollingBack); + assertAllowed(rollingBack, rolledBack); + assertRejected(external, verifying); + assertRejected(failed, failed); + assertRejected(rolledBack, rolledBack); + } + + @Test + void verificationSuccessExitMatchesApplyMode() { + MigrationOperationSnapshot managedVerifying = snapshot(MigrationOperationState.RUNNING, + MigrationStage.VERIFYING, 100, STARTED, null, VerificationState.RUNNING, + null, 1000, false, false, false); + MigrationOperationSnapshot managedExternal = snapshot(MigrationOperationState.AWAITING_EXTERNAL_APPLY, + MigrationStage.AWAITING_EXTERNAL_APPLY, 100, STARTED, null, VerificationState.SUCCEEDED, + null, 0, false, false, true); + MigrationOperationSnapshot externalVerifying = external(MigrationOperationState.RUNNING, + MigrationStage.VERIFYING, VerificationState.RUNNING, null, false, null); + MigrationOperationSnapshot externalReady = external(MigrationOperationState.READY_TO_ACTIVATE, + MigrationStage.READY_TO_ACTIVATE, VerificationState.SUCCEEDED, null, false, null); + + assertRejected(managedVerifying, managedExternal); + assertRejected(externalVerifying, externalReady); + } + + @Test + void failureCodesAndRollbackEntryMatchTheirSourceStage() { + MigrationOperationSnapshot pending = snapshot(MigrationOperationState.PENDING, MigrationStage.QUEUED, + 0, null, null, VerificationState.PENDING, null, 1000, false, false, false); + MigrationOperationSnapshot copying = snapshot(MigrationOperationState.RUNNING, MigrationStage.COPYING, + 25, STARTED, null, VerificationState.PENDING, null, 1000, false, false, false); + MigrationOperationSnapshot verifying = snapshot(MigrationOperationState.RUNNING, MigrationStage.VERIFYING, + 100, STARTED, null, VerificationState.RUNNING, null, 1000, false, false, false); + MigrationOperationSnapshot activating = snapshot(MigrationOperationState.RUNNING, MigrationStage.ACTIVATING, + 100, STARTED, null, VerificationState.SUCCEEDED, null, 1000, false, false, false); + MigrationOperationSnapshot restart = snapshot(MigrationOperationState.AWAITING_RESTART, + MigrationStage.AWAITING_RESTART, 100, STARTED, null, VerificationState.SUCCEEDED, + null, 1000, false, true, false); + MigrationOperationSnapshot rollingBack = rollingBack(MigrationRollbackOrigin.RESTART_FAILURE); + MigrationOperationSnapshot rolledBack = rolledBack(MigrationRollbackOrigin.RESTART_FAILURE, + SetupErrorCode.RESTART_FAILED); + + assertAllowed(pending, failed(SetupErrorCode.MIGRATION_COPY_FAILED)); + assertAllowed(copying, failed(SetupErrorCode.MIGRATION_COPY_FAILED)); + assertAllowed(verifying, failed(SetupErrorCode.MIGRATION_VERIFICATION_FAILED)); + assertAllowed(activating, failed(SetupErrorCode.MIGRATION_ACTIVATION_FAILED)); + assertAllowed(restart, failed(SetupErrorCode.RESTART_FAILED)); + assertRejected(copying, failed(SetupErrorCode.MIGRATION_ACTIVATION_FAILED)); + assertRejected(verifying, failed(SetupErrorCode.MIGRATION_ACTIVATION_FAILED)); + assertRejected(activating, failed(SetupErrorCode.MIGRATION_VERIFICATION_FAILED)); + assertRejected(restart, failed(SetupErrorCode.MIGRATION_ACTIVATION_FAILED)); + assertRejected(restart, rolledBack); + assertAllowed(restart, rollingBack); + assertAllowed(rollingBack, rolledBack); + } + + @Test + void rollbackOriginFreezesVerificationAndTerminalFailureSource() { + MigrationOperationSnapshot ready = snapshot(MigrationOperationState.READY_TO_ACTIVATE, + MigrationStage.READY_TO_ACTIVATE, 100, STARTED, null, VerificationState.SUCCEEDED, + null, 0, true, false, false); + MigrationOperationSnapshot restart = snapshot(MigrationOperationState.AWAITING_RESTART, + MigrationStage.AWAITING_RESTART, 100, STARTED, null, VerificationState.SUCCEEDED, + null, 1000, false, true, false); + MigrationOperationSnapshot activationRollback = rollingBack(MigrationRollbackOrigin.ACTIVATION_FAILURE); + MigrationOperationSnapshot restartRollback = rollingBack(MigrationRollbackOrigin.RESTART_FAILURE); + MigrationOperationSnapshot verificationRollback = rollingBack(MigrationRollbackOrigin.VERIFICATION_FAILURE); + + assertAllowed(ready, activationRollback); + assertRejected(ready, restartRollback); + assertAllowed(restart, restartRollback); + assertRejected(restart, activationRollback); + assertRejected(verificationRollback, rolledBack(MigrationRollbackOrigin.ACTIVATION_FAILURE, + SetupErrorCode.MIGRATION_ACTIVATION_FAILED)); + assertThatThrownBy(() -> rolledBack(MigrationRollbackOrigin.ACTIVATION_FAILURE, + SetupErrorCode.RESTART_FAILED)).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> rolledBack(MigrationRollbackOrigin.RESTART_FAILURE, + SetupErrorCode.MIGRATION_VERIFICATION_FAILED)).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> rolledBack(MigrationRollbackOrigin.VERIFICATION_FAILURE, + SetupErrorCode.MIGRATION_ACTIVATION_FAILED)).isInstanceOf(IllegalArgumentException.class); + } + + @Test + void rejectsSkippingCopyVerificationAndActivationStages() { + MigrationOperationSnapshot copying = snapshot(MigrationOperationState.RUNNING, MigrationStage.COPYING, + 25, STARTED, null, VerificationState.PENDING, null, 1000, false, false, false); + MigrationOperationSnapshot ready = snapshot(MigrationOperationState.READY_TO_ACTIVATE, + MigrationStage.READY_TO_ACTIVATE, 100, STARTED, null, VerificationState.SUCCEEDED, + null, 0, true, false, false); + MigrationOperationSnapshot restart = snapshot(MigrationOperationState.AWAITING_RESTART, + MigrationStage.AWAITING_RESTART, 100, STARTED, null, VerificationState.SUCCEEDED, + null, 1000, false, true, false); + MigrationOperationSnapshot succeeded = snapshot(MigrationOperationState.SUCCEEDED, MigrationStage.COMPLETED, + 100, STARTED, COMPLETED, VerificationState.SUCCEEDED, null, 0, false, false, false); + MigrationOperationSnapshot verifying = snapshot(MigrationOperationState.RUNNING, MigrationStage.VERIFYING, + 100, STARTED, null, VerificationState.RUNNING, null, 1000, false, false, false); + + assertRejected(copying, ready); + assertRejected(copying, restart); + assertRejected(copying, succeeded); + assertRejected(ready, verifying); + } + + private MigrationOperationSnapshot external( + MigrationOperationState state, MigrationStage stage, VerificationState verification, + SetupErrorCode error, boolean externalRequired, Instant completedAt) { + return new MigrationOperationSnapshot("migration-1", state, MigrationTarget.POSTGRESQL, + ApplyMode.EXTERNAL_APPLY, stage, 100, CREATED, STARTED, completedAt, verification, + error, null, state == MigrationOperationState.RUNNING ? 1000 : 0, + state == MigrationOperationState.READY_TO_ACTIVATE, false, externalRequired); + } + + private MigrationOperationSnapshot snapshot( + MigrationOperationState state, MigrationStage stage, int progress, Instant startedAt, + Instant completedAt, VerificationState verification, SetupErrorCode error, long pollMillis, + boolean activation, boolean restart, boolean external) { + return new MigrationOperationSnapshot("migration-1", state, MigrationTarget.MYSQL, + ApplyMode.MANAGED_WRITE, stage, progress, CREATED, startedAt, completedAt, + verification, error, null, pollMillis, activation, restart, external); + } + + private MigrationOperationSnapshot failed(SetupErrorCode errorCode) { + int progress = errorCode == SetupErrorCode.MIGRATION_COPY_FAILED ? 25 : 100; + VerificationState verification = switch (errorCode) { + case MIGRATION_COPY_FAILED -> VerificationState.PENDING; + case MIGRATION_VERIFICATION_FAILED -> VerificationState.FAILED; + default -> VerificationState.SUCCEEDED; + }; + return snapshot(MigrationOperationState.FAILED, MigrationStage.FAILED, progress, STARTED, + COMPLETED, verification, errorCode, 0, false, false, false); + } + + private MigrationOperationSnapshot rollingBack(MigrationRollbackOrigin origin) { + return new MigrationOperationSnapshot("migration-1", MigrationOperationState.RUNNING, + MigrationTarget.MYSQL, ApplyMode.MANAGED_WRITE, MigrationStage.ROLLING_BACK, + 100, CREATED, STARTED, null, origin.verificationState(), null, origin, + 1000, false, false, false); + } + + private MigrationOperationSnapshot rolledBack( + MigrationRollbackOrigin origin, SetupErrorCode errorCode) { + return new MigrationOperationSnapshot("migration-1", MigrationOperationState.ROLLED_BACK, + MigrationTarget.MYSQL, ApplyMode.MANAGED_WRITE, MigrationStage.ROLLED_BACK, + 100, CREATED, STARTED, COMPLETED, origin.verificationState(), errorCode, origin, + 0, false, false, false); + } + + private void assertAllowed(MigrationOperationSnapshot current, MigrationOperationSnapshot next) { + assertThatCode(() -> policy.requireAllowed(current, next)).doesNotThrowAnyException(); + } + + private void assertRejected(MigrationOperationSnapshot current, MigrationOperationSnapshot next) { + assertThatThrownBy(() -> policy.requireAllowed(current, next)) + .isInstanceOf(MigrationOperationStoreException.class); + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupTransitionSplitLockTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupTransitionSplitLockTest.java index 4f0c499f89..25b936978f 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupTransitionSplitLockTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/runtime/SetupTransitionSplitLockTest.java @@ -26,6 +26,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.apache.hertzbeat.common.runtime.RuntimeMode; import org.apache.hertzbeat.manager.setup.runtime.SetupTransitionIntentStore.Intent; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFileLock; import org.apache.hertzbeat.startup.runtime.HertzBeatStartupCoordinator; import org.apache.hertzbeat.startup.runtime.RunningApplicationContext; import org.apache.hertzbeat.startup.runtime.StartupContextLauncher; @@ -55,7 +56,7 @@ class SetupTransitionSplitLockTest { FileSetupTransitionIntentStore.COMPLETION_RELATIVE_PATH); FileSetupTransitionIntentStore staleContext = new FileSetupTransitionIntentStore( installationRoot, ignored -> { }, - new FileSetupTransitionIntentLock( + new SecureSetupFileLock( installationRoot, "data/config/.setup-transition-stale.lock"), (path, present) -> { if (path.equals(completionMarker) && !present @@ -80,7 +81,7 @@ class SetupTransitionSplitLockTest { private FileSetupTransitionIntentStore storeWithLock(String identity) { return new FileSetupTransitionIntentStore( installationRoot, ignored -> { }, - new FileSetupTransitionIntentLock( + new SecureSetupFileLock( installationRoot, "data/config/.setup-transition-" + identity + ".lock"), FileSetupTransitionIntentStore.MarkerObservation.NONE); } From d0b4c194eab24caf9dea55503e830980fbd39b34 Mon Sep 17 00:00:00 2001 From: Logic Date: Sun, 9 Aug 2026 23:47:44 +0800 Subject: [PATCH 35/71] Gate metadata writes during maintenance --- hertzbeat-common-spring/pom.xml | 5 + .../MetadataWriteAdmissionAdvisor.java | 116 ++++ .../MetadataWriteAdmissionConfiguration.java | 50 ++ .../MetadataWriteAdmissionCoordinator.java | 151 ++++++ .../MetadataWriteAdmissionErrorCode.java | 28 + .../MetadataWriteAdmissionException.java | 61 +++ .../MetadataWriteAdmissionPhase.java | 15 + .../MetadataWriteAdmissionSnapshot.java | 16 + .../MetadataWriteMaintenanceLease.java | 35 ++ ...ngDataWriteAdmissionBeanPostProcessor.java | 88 +++ .../TransactionCompletionPermitRegistry.java | 80 +++ ...MetadataWriteAdmissionIntegrationTest.java | 504 ++++++++++++++++++ ...ansactionCompletionPermitRegistryTest.java | 39 ++ .../setup/api/SetupExceptionHandler.java | 9 + .../support/GlobalExceptionHandler.java | 9 + ...MetadataWriteMaintenanceErrorResponse.java | 32 ++ ...MetadataWriteAdmissionHttpMappingTest.java | 43 ++ .../setup/api/DeploymentControllerTest.java | 14 + .../api/SetupExceptionHandlerLoggingTest.java | 16 + .../support/GlobalExceptionHandlerTest.java | 19 + ...adataWriteAdmissionStartupContextTest.java | 122 +++++ hertzbeat-warehouse/pom.xml | 5 + .../warehouse/store/DataStorageDispatch.java | 56 +- .../JdbcMonitorStatusMetadataWriter.java | 47 ++ .../store/metadata/MonitorAvailability.java | 14 + .../metadata/MonitorStatusMetadataWriter.java | 15 + .../store/DataStorageDispatchContextTest.java | 6 +- ...rageDispatchMaintenanceContinuityTest.java | 135 +++++ .../store/DataStorageDispatchStatusTest.java | 15 +- ...orStatusMetadataWriterIntegrationTest.java | 125 +++++ 30 files changed, 1826 insertions(+), 44 deletions(-) create mode 100644 hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionAdvisor.java create mode 100644 hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionConfiguration.java create mode 100644 hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionCoordinator.java create mode 100644 hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionErrorCode.java create mode 100644 hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionException.java create mode 100644 hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionPhase.java create mode 100644 hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionSnapshot.java create mode 100644 hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteMaintenanceLease.java create mode 100644 hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/SpringDataWriteAdmissionBeanPostProcessor.java create mode 100644 hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/TransactionCompletionPermitRegistry.java create mode 100644 hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionIntegrationTest.java create mode 100644 hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/transaction/TransactionCompletionPermitRegistryTest.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/support/MetadataWriteMaintenanceErrorResponse.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionHttpMappingTest.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/MetadataWriteAdmissionStartupContextTest.java create mode 100644 hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/metadata/JdbcMonitorStatusMetadataWriter.java create mode 100644 hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/metadata/MonitorAvailability.java create mode 100644 hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/metadata/MonitorStatusMetadataWriter.java create mode 100644 hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatchMaintenanceContinuityTest.java create mode 100644 hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/metadata/MonitorStatusMetadataWriterIntegrationTest.java diff --git a/hertzbeat-common-spring/pom.xml b/hertzbeat-common-spring/pom.xml index 7aafb4ddd1..048c54ea7e 100644 --- a/hertzbeat-common-spring/pom.xml +++ b/hertzbeat-common-spring/pom.xml @@ -68,6 +68,11 @@ springdoc-openapi-starter-webmvc-ui provided + + com.h2database + h2 + test + diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionAdvisor.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionAdvisor.java new file mode 100644 index 0000000000..702a2ece46 --- /dev/null +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionAdvisor.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.common.transaction; + +import java.lang.reflect.Method; +import org.aopalliance.intercept.MethodInterceptor; +import org.springframework.aop.Pointcut; +import org.springframework.aop.support.AbstractPointcutAdvisor; +import org.springframework.aop.support.AopUtils; +import org.springframework.aop.support.StaticMethodMatcherPointcut; +import org.springframework.core.Ordered; +import org.springframework.data.repository.Repository; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.interceptor.TransactionAttribute; +import org.springframework.transaction.interceptor.TransactionAttributeSource; +import org.springframework.transaction.support.TransactionSynchronizationManager; + +/** Admits writable transactional boundaries before Spring opens their transactions. */ +public final class MetadataWriteAdmissionAdvisor extends AbstractPointcutAdvisor { + + private static final int ADVISOR_ORDER = Ordered.HIGHEST_PRECEDENCE + 100; + + private final TransactionAttributeSource transactionAttributes; + private final MetadataWriteAdmissionCoordinator coordinator; + private final TransactionCompletionPermitRegistry transactionPermits; + private final boolean repositoryAttributes; + private final Pointcut pointcut = new TransactionAttributePointcut(); + private final MethodInterceptor advice = this::invoke; + + MetadataWriteAdmissionAdvisor( + TransactionAttributeSource transactionAttributes, + MetadataWriteAdmissionCoordinator coordinator, + TransactionCompletionPermitRegistry transactionPermits, + boolean repositoryAttributes) { + this.transactionAttributes = transactionAttributes; + this.coordinator = coordinator; + this.transactionPermits = transactionPermits; + this.repositoryAttributes = repositoryAttributes; + } + + @Override + public Pointcut getPointcut() { + return pointcut; + } + + @Override + public MethodInterceptor getAdvice() { + return advice; + } + + @Override + public int getOrder() { + return ADVISOR_ORDER; + } + + private Object invoke(org.aopalliance.intercept.MethodInvocation invocation) throws Throwable { + TransactionAttribute attribute = resolveAttribute(invocation.getMethod(), invocation.getThis()); + if (attribute == null || attribute.isReadOnly()) { + return invocation.proceed(); + } + if (transactionPermits.hasPermit()) { + return invocation.proceed(); + } + if (joinsExistingPhysicalTransaction(attribute)) { + MetadataWriteAdmissionCoordinator.TransactionPermit permit = coordinator.admitWritableTransaction(); + transactionPermits.bind(permit); + transactionPermits.beginInvocation(); + try { + return invocation.proceed(); + } finally { + transactionPermits.endInvocation(); + } + } + try (MetadataWriteAdmissionCoordinator.TransactionPermit permit = coordinator.admitWritableTransaction()) { + transactionPermits.beginInvocation(); + try { + return invocation.proceed(); + } finally { + transactionPermits.endInvocation(); + } + } + } + + private TransactionAttribute resolveAttribute(Method method, Object target) { + Class targetClass = target == null ? method.getDeclaringClass() : AopUtils.getTargetClass(target); + return transactionAttributes.getTransactionAttribute(method, targetClass); + } + + private boolean joinsExistingPhysicalTransaction(TransactionAttribute attribute) { + if (!TransactionSynchronizationManager.isActualTransactionActive()) { + return false; + } + return switch (attribute.getPropagationBehavior()) { + case TransactionDefinition.PROPAGATION_REQUIRED, + TransactionDefinition.PROPAGATION_SUPPORTS, + TransactionDefinition.PROPAGATION_MANDATORY, + TransactionDefinition.PROPAGATION_NESTED -> true; + default -> false; + }; + } + + private final class TransactionAttributePointcut extends StaticMethodMatcherPointcut { + + @Override + public boolean matches(Method method, Class targetClass) { + return (repositoryAttributes || !Repository.class.isAssignableFrom(targetClass)) + && transactionAttributes.getTransactionAttribute(method, targetClass) != null; + } + } + +} diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionConfiguration.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionConfiguration.java new file mode 100644 index 0000000000..fcb623c13b --- /dev/null +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionConfiguration.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.common.transaction; + +import org.springframework.beans.factory.config.BeanDefinition; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Role; +import org.springframework.transaction.interceptor.TransactionAttributeSource; + +/** Spring wiring for process-local metadata write admission. */ +@Configuration(proxyBeanMethods = false) +@Role(BeanDefinition.ROLE_INFRASTRUCTURE) +public class MetadataWriteAdmissionConfiguration { + + @Bean + @Role(BeanDefinition.ROLE_INFRASTRUCTURE) + MetadataWriteAdmissionCoordinator metadataWriteAdmissionCoordinator() { + return new MetadataWriteAdmissionCoordinator(); + } + + @Bean + @Role(BeanDefinition.ROLE_INFRASTRUCTURE) + TransactionCompletionPermitRegistry transactionCompletionPermitRegistry() { + return new TransactionCompletionPermitRegistry(); + } + + @Bean + @Role(BeanDefinition.ROLE_INFRASTRUCTURE) + MetadataWriteAdmissionAdvisor metadataWriteAdmissionAdvisor( + TransactionAttributeSource transactionAttributeSource, + MetadataWriteAdmissionCoordinator coordinator, + TransactionCompletionPermitRegistry transactionPermits) { + return new MetadataWriteAdmissionAdvisor( + transactionAttributeSource, coordinator, transactionPermits, false); + } + + @Bean + @Role(BeanDefinition.ROLE_INFRASTRUCTURE) + static SpringDataWriteAdmissionBeanPostProcessor springDataWriteAdmissionBeanPostProcessor( + MetadataWriteAdmissionCoordinator coordinator, + TransactionCompletionPermitRegistry transactionPermits) { + return new SpringDataWriteAdmissionBeanPostProcessor(coordinator, transactionPermits); + } +} diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionCoordinator.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionCoordinator.java new file mode 100644 index 0000000000..173ccdb4be --- /dev/null +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionCoordinator.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.common.transaction; + +import java.time.Duration; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; + +/** Coordinates process-local writable transaction admission and maintenance drain. */ +public final class MetadataWriteAdmissionCoordinator { + + private final ReentrantLock lock = new ReentrantLock(); + private final Condition noActiveWrites = lock.newCondition(); + private MetadataWriteAdmissionPhase phase = MetadataWriteAdmissionPhase.OPEN; + private String operationId; + private long epoch; + private Object leaseToken; + private int activeWrites; + + /** Drain admitted writes and enter maintenance for one operation. */ + public MetadataWriteMaintenanceLease acquire(String requestedOperationId, Duration timeout) { + requireValid(requestedOperationId, timeout); + long timeoutNanos = toNanos(timeout); + lock.lock(); + try { + if (phase != MetadataWriteAdmissionPhase.OPEN) { + throw MetadataWriteAdmissionException.operationConflict(); + } + phase = MetadataWriteAdmissionPhase.DRAINING; + operationId = requestedOperationId; + long currentEpoch = ++epoch; + Object currentToken = new Object(); + leaseToken = currentToken; + long remainingNanos = timeoutNanos; + while (activeWrites > 0) { + if (remainingNanos <= 0) { + reopen(currentEpoch, currentToken); + throw MetadataWriteAdmissionException.drainTimeout(); + } + try { + remainingNanos = noActiveWrites.awaitNanos(remainingNanos); + } catch (InterruptedException exception) { + reopen(currentEpoch, currentToken); + Thread.currentThread().interrupt(); + throw MetadataWriteAdmissionException.acquisitionInterrupted(); + } + } + phase = MetadataWriteAdmissionPhase.ACTIVE; + return new MetadataWriteMaintenanceLease(this, requestedOperationId, currentEpoch, currentToken); + } finally { + lock.unlock(); + } + } + + /** Return a consistent view without exposing the lease capability. */ + public MetadataWriteAdmissionSnapshot snapshot() { + lock.lock(); + try { + return new MetadataWriteAdmissionSnapshot(phase, operationId, epoch, activeWrites); + } finally { + lock.unlock(); + } + } + + TransactionPermit admitWritableTransaction() { + lock.lock(); + try { + if (phase != MetadataWriteAdmissionPhase.OPEN) { + throw MetadataWriteAdmissionException.metadataWritesPaused(); + } + activeWrites++; + return new TransactionPermit(this); + } finally { + lock.unlock(); + } + } + + void release(String releasedOperationId, long releasedEpoch, Object releasedToken) { + lock.lock(); + try { + if (phase == MetadataWriteAdmissionPhase.ACTIVE + && epoch == releasedEpoch + && operationId.equals(releasedOperationId) + && leaseToken == releasedToken) { + phase = MetadataWriteAdmissionPhase.OPEN; + operationId = null; + leaseToken = null; + } + } finally { + lock.unlock(); + } + } + + private void releaseWritableTransaction() { + lock.lock(); + try { + activeWrites--; + if (activeWrites == 0) { + noActiveWrites.signalAll(); + } + } finally { + lock.unlock(); + } + } + + private void reopen(long failedEpoch, Object failedToken) { + if (epoch == failedEpoch && leaseToken == failedToken && phase == MetadataWriteAdmissionPhase.DRAINING) { + phase = MetadataWriteAdmissionPhase.OPEN; + operationId = null; + leaseToken = null; + } + } + + private void requireValid(String requestedOperationId, Duration timeout) { + if (requestedOperationId == null || requestedOperationId.isBlank() + || timeout == null || timeout.isNegative()) { + throw MetadataWriteAdmissionException.invalidRequest(); + } + } + + private long toNanos(Duration timeout) { + try { + return timeout.toNanos(); + } catch (ArithmeticException exception) { + throw MetadataWriteAdmissionException.invalidRequest(); + } + } + + static final class TransactionPermit implements AutoCloseable { + + private final MetadataWriteAdmissionCoordinator coordinator; + private boolean closed; + + private TransactionPermit(MetadataWriteAdmissionCoordinator coordinator) { + this.coordinator = coordinator; + } + + @Override + public void close() { + if (!closed) { + closed = true; + coordinator.releaseWritableTransaction(); + } + } + } +} diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionErrorCode.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionErrorCode.java new file mode 100644 index 0000000000..d568d5bbaf --- /dev/null +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionErrorCode.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.common.transaction; + +/** Stable, secret-free failure classifications for metadata write admission. */ +public enum MetadataWriteAdmissionErrorCode { + MAINTENANCE_ACTIVE("metadata_writes_paused"), + OPERATION_CONFLICT("operation_conflict"), + DRAIN_TIMEOUT("drain_timeout"), + ACQUISITION_INTERRUPTED("acquisition_interrupted"), + INVALID_REQUEST("invalid_request"); + + private final String wireCode; + + MetadataWriteAdmissionErrorCode(String wireCode) { + this.wireCode = wireCode; + } + + /** Return the stable, safe code exposed at typed transport boundaries. */ + public String wireCode() { + return wireCode; + } +} diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionException.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionException.java new file mode 100644 index 0000000000..8e9c51f122 --- /dev/null +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionException.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.common.transaction; + +/** Safe admission failure that never exposes operation identifiers or persistence details. */ +public final class MetadataWriteAdmissionException extends RuntimeException { + + private static final String MAINTENANCE_MESSAGE = "Metadata writes are temporarily unavailable"; + private static final String CONFLICT_MESSAGE = "Metadata maintenance operation is already active"; + private static final String TIMEOUT_MESSAGE = "Metadata write drain timed out"; + private static final String INTERRUPTED_MESSAGE = "Metadata write drain was interrupted"; + private static final String INVALID_MESSAGE = "Metadata maintenance request is invalid"; + + private final MetadataWriteAdmissionErrorCode code; + + private MetadataWriteAdmissionException(MetadataWriteAdmissionErrorCode code, String message) { + super(message); + this.code = code; + } + + /** Return the stable machine-readable classification. */ + public MetadataWriteAdmissionErrorCode code() { + return code; + } + + /** Return the stable, secret-free message suitable for typed transport boundaries. */ + public String safeMessage() { + return getMessage(); + } + + /** Create the stable rejection used by typed metadata-write callers and tests. */ + public static MetadataWriteAdmissionException metadataWritesPaused() { + return new MetadataWriteAdmissionException( + MetadataWriteAdmissionErrorCode.MAINTENANCE_ACTIVE, MAINTENANCE_MESSAGE); + } + + static MetadataWriteAdmissionException operationConflict() { + return new MetadataWriteAdmissionException( + MetadataWriteAdmissionErrorCode.OPERATION_CONFLICT, CONFLICT_MESSAGE); + } + + static MetadataWriteAdmissionException drainTimeout() { + return new MetadataWriteAdmissionException( + MetadataWriteAdmissionErrorCode.DRAIN_TIMEOUT, TIMEOUT_MESSAGE); + } + + static MetadataWriteAdmissionException acquisitionInterrupted() { + return new MetadataWriteAdmissionException( + MetadataWriteAdmissionErrorCode.ACQUISITION_INTERRUPTED, INTERRUPTED_MESSAGE); + } + + static MetadataWriteAdmissionException invalidRequest() { + return new MetadataWriteAdmissionException( + MetadataWriteAdmissionErrorCode.INVALID_REQUEST, INVALID_MESSAGE); + } +} diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionPhase.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionPhase.java new file mode 100644 index 0000000000..9b7d72e780 --- /dev/null +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionPhase.java @@ -0,0 +1,15 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.common.transaction; + +/** Metadata write admission lifecycle. */ +public enum MetadataWriteAdmissionPhase { + OPEN, + DRAINING, + ACTIVE +} diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionSnapshot.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionSnapshot.java new file mode 100644 index 0000000000..fd19d60374 --- /dev/null +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionSnapshot.java @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.common.transaction; + +/** Immutable diagnostic projection of local metadata write admission state. */ +public record MetadataWriteAdmissionSnapshot( + MetadataWriteAdmissionPhase phase, + String operationId, + long epoch, + int activeWritableTransactions) { +} diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteMaintenanceLease.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteMaintenanceLease.java new file mode 100644 index 0000000000..407e032293 --- /dev/null +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/MetadataWriteMaintenanceLease.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.common.transaction; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** Capability that returns metadata write admission to OPEN when its matching epoch is released. */ +public final class MetadataWriteMaintenanceLease implements AutoCloseable { + + private final MetadataWriteAdmissionCoordinator coordinator; + private final String operationId; + private final long epoch; + private final Object token; + private final AtomicBoolean closed = new AtomicBoolean(); + + MetadataWriteMaintenanceLease( + MetadataWriteAdmissionCoordinator coordinator, String operationId, long epoch, Object token) { + this.coordinator = coordinator; + this.operationId = operationId; + this.epoch = epoch; + this.token = token; + } + + @Override + public void close() { + if (closed.compareAndSet(false, true)) { + coordinator.release(operationId, epoch, token); + } + } +} diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/SpringDataWriteAdmissionBeanPostProcessor.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/SpringDataWriteAdmissionBeanPostProcessor.java new file mode 100644 index 0000000000..6c4fc1e1a7 --- /dev/null +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/SpringDataWriteAdmissionBeanPostProcessor.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.common.transaction; + +import org.springframework.aop.Advisor; +import org.springframework.aop.framework.Advised; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.BeanInitializationException; +import org.springframework.beans.factory.config.BeanPostProcessor; +import org.springframework.core.Ordered; +import org.springframework.data.repository.Repository; +import org.springframework.transaction.interceptor.TransactionAttributeSource; +import org.springframework.transaction.interceptor.TransactionInterceptor; +import org.springframework.transaction.interceptor.TransactionalProxy; + +/** Inserts write admission into each existing Spring Data transaction proxy. */ +final class SpringDataWriteAdmissionBeanPostProcessor implements BeanPostProcessor, Ordered { + + private final MetadataWriteAdmissionCoordinator coordinator; + private final TransactionCompletionPermitRegistry transactionPermits; + + SpringDataWriteAdmissionBeanPostProcessor( + MetadataWriteAdmissionCoordinator coordinator, + TransactionCompletionPermitRegistry transactionPermits) { + this.coordinator = coordinator; + this.transactionPermits = transactionPermits; + } + + @Override + public int getOrder() { + return Ordered.LOWEST_PRECEDENCE; + } + + @Override + public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException { + if (!(bean instanceof Repository) + || !(bean instanceof TransactionalProxy) + || !(bean instanceof Advised advised)) { + return bean; + } + if (hasAdmissionAdvisor(advised)) { + return bean; + } + if (advised.isFrozen()) { + throw new BeanInitializationException("Spring Data transaction proxy is frozen"); + } + int transactionAdvisorIndex = transactionAdvisorIndex(advised); + TransactionInterceptor interceptor = (TransactionInterceptor) advised + .getAdvisors()[transactionAdvisorIndex].getAdvice(); + TransactionAttributeSource attributes = interceptor.getTransactionAttributeSource(); + if (attributes == null) { + throw new BeanInitializationException("Spring Data transaction attributes are unavailable"); + } + advised.addAdvisor(transactionAdvisorIndex, + new MetadataWriteAdmissionAdvisor(attributes, coordinator, transactionPermits, true)); + return bean; + } + + private boolean hasAdmissionAdvisor(Advised advised) { + for (Advisor advisor : advised.getAdvisors()) { + if (advisor instanceof MetadataWriteAdmissionAdvisor) { + return true; + } + } + return false; + } + + private int transactionAdvisorIndex(Advised advised) { + int found = -1; + for (int index = 0; index < advised.getAdvisors().length; index++) { + if (advised.getAdvisors()[index].getAdvice() instanceof TransactionInterceptor) { + if (found >= 0) { + throw new BeanInitializationException("Spring Data transaction advisor is ambiguous"); + } + found = index; + } + } + if (found < 0) { + throw new BeanInitializationException("Spring Data transaction advisor is unavailable"); + } + return found; + } +} diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/TransactionCompletionPermitRegistry.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/TransactionCompletionPermitRegistry.java new file mode 100644 index 0000000000..5ef40c49ce --- /dev/null +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/transaction/TransactionCompletionPermitRegistry.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.common.transaction; + +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; + +/** Binds a writable admission permit to the current physical transaction completion. */ +final class TransactionCompletionPermitRegistry { + + private final Object resourceKey = new Object(); + private final SynchronizationRegistrar registrar; + private final ThreadLocal invocationPermit = new ThreadLocal<>(); + + TransactionCompletionPermitRegistry() { + this(TransactionSynchronizationManager::registerSynchronization); + } + + TransactionCompletionPermitRegistry(SynchronizationRegistrar registrar) { + this.registrar = registrar; + } + + boolean hasPermit() { + return invocationPermit.get() != null || TransactionSynchronizationManager.hasResource(resourceKey); + } + + void beginInvocation() { + invocationPermit.set(Boolean.TRUE); + } + + void endInvocation() { + invocationPermit.remove(); + } + + void bind(MetadataWriteAdmissionCoordinator.TransactionPermit permit) { + if (!TransactionSynchronizationManager.isActualTransactionActive() + || !TransactionSynchronizationManager.isSynchronizationActive()) { + permit.close(); + throw new IllegalStateException("Transaction synchronization is unavailable"); + } + boolean bound = false; + try { + TransactionSynchronizationManager.bindResource(resourceKey, permit); + bound = true; + registrar.register(new PermitReleaseSynchronization(permit)); + } catch (RuntimeException | Error failure) { + if (bound) { + TransactionSynchronizationManager.unbindResourceIfPossible(resourceKey); + } + permit.close(); + throw failure; + } + } + + @FunctionalInterface + interface SynchronizationRegistrar { + + void register(TransactionSynchronization synchronization); + } + + private final class PermitReleaseSynchronization implements TransactionSynchronization { + + private final MetadataWriteAdmissionCoordinator.TransactionPermit permit; + + private PermitReleaseSynchronization(MetadataWriteAdmissionCoordinator.TransactionPermit permit) { + this.permit = permit; + } + + @Override + public void afterCompletion(int status) { + TransactionSynchronizationManager.unbindResourceIfPossible(resourceKey); + permit.close(); + } + } +} diff --git a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionIntegrationTest.java b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionIntegrationTest.java new file mode 100644 index 0000000000..6a3af11c71 --- /dev/null +++ b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionIntegrationTest.java @@ -0,0 +1,504 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.common.transaction; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.aop.support.AopUtils; +import org.springframework.aop.framework.Advised; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.jdbc.datasource.DriverManagerDataSource; +import org.springframework.orm.jpa.JpaTransactionManager; +import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; +import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.IllegalTransactionStateException; +import org.springframework.transaction.interceptor.BeanFactoryTransactionAttributeSourceAdvisor; +import org.springframework.transaction.interceptor.TransactionInterceptor; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; + +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = MetadataWriteAdmissionIntegrationTest.TestConfiguration.class) +class MetadataWriteAdmissionIntegrationTest { + + private final ExecutorService executor = Executors.newCachedThreadPool(); + + @Autowired + private MetadataWriteAdmissionCoordinator coordinator; + + @Autowired + private AdmissionService service; + + @Autowired + private AdmissionRepository repository; + + @Autowired + private MetadataWriteAdmissionAdvisor admissionAdvisor; + + @Autowired + private BeanFactoryTransactionAttributeSourceAdvisor transactionAdvisor; + + @AfterEach + void shutdownExecutor() { + executor.shutdownNow(); + } + + @BeforeEach + void clearRows() { + repository.deleteAll(); + } + + @Test + void drainWaitsForAdmittedTransactionCommitAndRollbackCompletion() throws Exception { + CountDownLatch commitStarted = new CountDownLatch(1); + CountDownLatch finishCommit = new CountDownLatch(1); + CountDownLatch afterCompletionEntered = new CountDownLatch(1); + CountDownLatch finishAfterCompletion = new CountDownLatch(1); + Future write = executor.submit(() -> service.holdWrite( + commitStarted, finishCommit, afterCompletionEntered, finishAfterCompletion, false)); + assertThat(commitStarted.await(2, TimeUnit.SECONDS)).isTrue(); + + Future acquiring = executor.submit( + () -> coordinator.acquire("operation-commit", Duration.ofSeconds(3))); + awaitPhase(MetadataWriteAdmissionPhase.DRAINING); + assertThat(acquiring.isDone()).isFalse(); + finishCommit.countDown(); + assertThat(afterCompletionEntered.await(2, TimeUnit.SECONDS)).isTrue(); + assertThat(coordinator.snapshot().activeWritableTransactions()).isEqualTo(1); + assertThat(acquiring.isDone()).isFalse(); + finishAfterCompletion.countDown(); + write.get(2, TimeUnit.SECONDS); + try (MetadataWriteMaintenanceLease ignored = acquiring.get(2, TimeUnit.SECONDS)) { + assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataWriteAdmissionPhase.ACTIVE); + } + assertThat(service.count()).isEqualTo(1); + + CountDownLatch rollbackStarted = new CountDownLatch(1); + CountDownLatch finishRollback = new CountDownLatch(1); + CountDownLatch rollbackCompletionEntered = new CountDownLatch(1); + CountDownLatch finishRollbackCompletion = new CountDownLatch(1); + Future rollback = executor.submit(() -> service.holdWrite( + rollbackStarted, finishRollback, rollbackCompletionEntered, finishRollbackCompletion, true)); + assertThat(rollbackStarted.await(2, TimeUnit.SECONDS)).isTrue(); + Future rollbackDrain = executor.submit( + () -> coordinator.acquire("operation-rollback", Duration.ofSeconds(3))); + awaitPhase(MetadataWriteAdmissionPhase.DRAINING); + finishRollback.countDown(); + assertThat(rollbackCompletionEntered.await(2, TimeUnit.SECONDS)).isTrue(); + assertThat(coordinator.snapshot().activeWritableTransactions()).isEqualTo(1); + assertThat(rollbackDrain.isDone()).isFalse(); + finishRollbackCompletion.countDown(); + assertThatThrownBy(() -> rollback.get(2, TimeUnit.SECONDS)).isInstanceOf(ExecutionException.class); + try (MetadataWriteMaintenanceLease ignored = rollbackDrain.get(2, TimeUnit.SECONDS)) { + assertThat(service.count()).isEqualTo(1); + } + } + + @Test + void drainingAndActiveRejectNewWritesButAllowReadOnlyAndRepositoryTransactions() throws Exception { + CountDownLatch admitted = new CountDownLatch(1); + CountDownLatch finish = new CountDownLatch(1); + Future existing = executor.submit(() -> service.holdWrite(admitted, finish, null, null, false)); + assertThat(admitted.await(2, TimeUnit.SECONDS)).isTrue(); + Future acquiring = executor.submit( + () -> coordinator.acquire("operation-gate", Duration.ofSeconds(3))); + awaitPhase(MetadataWriteAdmissionPhase.DRAINING); + + assertMaintenanceRejection(() -> service.write("draining")); + assertThat(service.count()).isZero(); + finish.countDown(); + existing.get(2, TimeUnit.SECONDS); + + try (MetadataWriteMaintenanceLease ignored = acquiring.get(2, TimeUnit.SECONDS)) { + assertMaintenanceRejection(() -> service.write("active")); + assertMaintenanceRejection(() -> repository.save(new AdmissionRow("repository"))); + assertMaintenanceRejection(() -> repository.deleteById(1L)); + assertThat(repository.count()).isEqualTo(1); + assertThat(repository.findAll()).hasSize(1); + assertThat(repository.declaredReadOnly()).hasSize(1); + assertMaintenanceRejection(repository::declaredWritable); + assertMaintenanceRejection(service::readOnlyThenNestedWrite); + } + service.write("open-again"); + assertThat(service.count()).isEqualTo(2); + } + + @Test + void nestedWritableBoundaryReusesOuterThreadPermitWhileDrainStarts() throws Exception { + CountDownLatch outerAdmitted = new CountDownLatch(1); + CountDownLatch invokeNested = new CountDownLatch(1); + Future outer = executor.submit(() -> service.outerThenNested(outerAdmitted, invokeNested)); + assertThat(outerAdmitted.await(2, TimeUnit.SECONDS)).isTrue(); + Future acquiring = executor.submit( + () -> coordinator.acquire("operation-nested", Duration.ofSeconds(3))); + awaitPhase(MetadataWriteAdmissionPhase.DRAINING); + invokeNested.countDown(); + outer.get(2, TimeUnit.SECONDS); + try (MetadataWriteMaintenanceLease ignored = acquiring.get(2, TimeUnit.SECONDS)) { + assertThat(repository.count()).isEqualTo(2); + } + } + + @Test + void requiredWriteJoinedToReadOnlyOuterTransactionKeepsPermitUntilPhysicalCompletion() throws Exception { + CountDownLatch nestedReturned = new CountDownLatch(1); + CountDownLatch finishOuter = new CountDownLatch(1); + Future outer = executor.submit(() -> service.readOnlyOuterHoldingAfterNestedWrite( + nestedReturned, finishOuter)); + assertThat(nestedReturned.await(2, TimeUnit.SECONDS)).isTrue(); + assertThat(coordinator.snapshot().activeWritableTransactions()).isEqualTo(1); + + Future acquiring = executor.submit( + () -> coordinator.acquire("operation-read-only-outer", Duration.ofSeconds(3))); + awaitPhase(MetadataWriteAdmissionPhase.DRAINING); + assertThat(acquiring.isDone()).isFalse(); + assertThat(coordinator.snapshot().activeWritableTransactions()).isEqualTo(1); + + finishOuter.countDown(); + outer.get(2, TimeUnit.SECONDS); + try (MetadataWriteMaintenanceLease ignored = acquiring.get(2, TimeUnit.SECONDS)) { + assertThat(repository.count()).isEqualTo(1); + } + } + + @Test + void timeoutRestoresOpenAndLeaseEpochPreventsStaleOrConcurrentRelease() throws Exception { + CountDownLatch admitted = new CountDownLatch(1); + CountDownLatch finish = new CountDownLatch(1); + Future existing = executor.submit(() -> service.holdWrite(admitted, finish, null, null, false)); + assertThat(admitted.await(2, TimeUnit.SECONDS)).isTrue(); + + assertThatThrownBy(() -> coordinator.acquire("operation-timeout", Duration.ofMillis(100))) + .isInstanceOfSatisfying(MetadataWriteAdmissionException.class, + error -> assertThat(error.code()).isEqualTo(MetadataWriteAdmissionErrorCode.DRAIN_TIMEOUT)); + assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataWriteAdmissionPhase.OPEN); + service.write("accepted-after-timeout"); + finish.countDown(); + existing.get(2, TimeUnit.SECONDS); + + MetadataWriteMaintenanceLease stale = coordinator.acquire("operation-one", Duration.ofSeconds(1)); + assertConflict(() -> coordinator.acquire("operation-one", Duration.ofSeconds(1))); + assertConflict(() -> coordinator.acquire("operation-other", Duration.ofSeconds(1))); + stale.close(); + MetadataWriteMaintenanceLease current = coordinator.acquire("operation-two", Duration.ofSeconds(1)); + stale.close(); + assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataWriteAdmissionPhase.ACTIVE); + assertThat(coordinator.snapshot().operationId()).isEqualTo("operation-two"); + current.close(); + current.close(); + assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataWriteAdmissionPhase.OPEN); + } + + @Test + void interruptedDrainReopensAdmissionAndTransactionStartFailureDoesNotLeakPermit() throws Exception { + assertThatThrownBy(service::mandatoryWrite) + .isInstanceOf(IllegalTransactionStateException.class); + assertThat(coordinator.snapshot().activeWritableTransactions()).isZero(); + + CountDownLatch admitted = new CountDownLatch(1); + CountDownLatch finish = new CountDownLatch(1); + Future existing = executor.submit(() -> service.holdWrite(admitted, finish, null, null, false)); + assertThat(admitted.await(2, TimeUnit.SECONDS)).isTrue(); + AtomicReference failure = new AtomicReference<>(); + CountDownLatch interrupted = new CountDownLatch(1); + Future acquiring = executor.submit(() -> { + try { + coordinator.acquire("operation-interrupted", Duration.ofSeconds(30)); + } catch (MetadataWriteAdmissionException exception) { + failure.set(exception); + } finally { + interrupted.countDown(); + } + }); + awaitPhase(MetadataWriteAdmissionPhase.DRAINING); + acquiring.cancel(true); + assertThat(interrupted.await(2, TimeUnit.SECONDS)).isTrue(); + assertThat(failure.get().code()).isEqualTo(MetadataWriteAdmissionErrorCode.ACQUISITION_INTERRUPTED); + assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataWriteAdmissionPhase.OPEN); + assertThat(coordinator.snapshot().activeWritableTransactions()).isEqualTo(1); + finish.countDown(); + existing.get(2, TimeUnit.SECONDS); + } + + @Test + void validatesOperationAndDurationWithoutLeakingRawFailures() { + assertInvalid(() -> coordinator.acquire(" ", Duration.ofSeconds(1))); + assertInvalid(() -> coordinator.acquire("operation", null)); + assertInvalid(() -> coordinator.acquire("operation", Duration.ofSeconds(-1))); + assertInvalid(() -> coordinator.acquire("operation", Duration.ofSeconds(Long.MAX_VALUE))); + + MetadataWriteMaintenanceLease lease = coordinator.acquire("zero-wait", Duration.ZERO); + lease.close(); + } + + @Test + void admissionAdvisorUsesTransactionAttributesAndRunsOutsideTransactionInterceptor() throws Exception { + assertThat(((Advised) service).getAdvisors()).contains(admissionAdvisor); + assertThat(admissionAdvisor.getOrder()).isLessThan(transactionAdvisor.getOrder()); + assertThat(admissionAdvisor.getPointcut().getMethodMatcher().matches( + AdmissionService.class.getMethod("write", String.class), AdmissionService.class)).isTrue(); + assertThat(AopUtils.isAopProxy(repository)).isTrue(); + Advised repositoryProxy = (Advised) repository; + assertThat(AopUtils.isAopProxy(repositoryProxy.getTargetSource().getTarget())).isFalse(); + assertThat(Arrays.stream(repositoryProxy.getAdvisors()) + .filter(MetadataWriteAdmissionAdvisor.class::isInstance)).hasSize(1); + assertThat(Arrays.stream(repositoryProxy.getAdvisors()) + .filter(advisor -> advisor.getAdvice() instanceof TransactionInterceptor)).hasSize(1); + + MetadataWriteAdmissionSnapshot observed = service.observeAdmissionInsideTransaction(); + assertThat(observed.phase()).isEqualTo(MetadataWriteAdmissionPhase.OPEN); + assertThat(observed.activeWritableTransactions()).isEqualTo(1); + } + + private void awaitPhase(MetadataWriteAdmissionPhase phase) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (coordinator.snapshot().phase() != phase && System.nanoTime() < deadline) { + Thread.onSpinWait(); + } + assertThat(coordinator.snapshot().phase()).isEqualTo(phase); + } + + private void assertMaintenanceRejection(Runnable write) { + assertThatThrownBy(write::run) + .isInstanceOfSatisfying(MetadataWriteAdmissionException.class, error -> { + assertThat(error.code()).isEqualTo(MetadataWriteAdmissionErrorCode.MAINTENANCE_ACTIVE); + assertThat(error.getMessage()).isEqualTo("Metadata writes are temporarily unavailable"); + assertThat(error.getCause()).isNull(); + }); + } + + private void assertConflict(ThrowingAcquire acquire) { + assertThatThrownBy(acquire::run) + .isInstanceOfSatisfying(MetadataWriteAdmissionException.class, + error -> assertThat(error.code()).isEqualTo(MetadataWriteAdmissionErrorCode.OPERATION_CONFLICT)); + } + + private void assertInvalid(ThrowingAcquire acquire) { + assertThatThrownBy(acquire::run) + .isInstanceOfSatisfying(MetadataWriteAdmissionException.class, + error -> assertThat(error.code()).isEqualTo(MetadataWriteAdmissionErrorCode.INVALID_REQUEST)); + } + + @FunctionalInterface + private interface ThrowingAcquire { + void run(); + } + + @Configuration(proxyBeanMethods = false) + @EnableTransactionManagement(order = 200) + @EnableJpaRepositories(considerNestedRepositories = true, + basePackageClasses = MetadataWriteAdmissionIntegrationTest.class) + @Import(MetadataWriteAdmissionConfiguration.class) + static class TestConfiguration { + + @Bean + DriverManagerDataSource dataSource() { + return new DriverManagerDataSource("jdbc:h2:mem:metadata-admission;DB_CLOSE_DELAY=-1", "sa", ""); + } + + @Bean + LocalContainerEntityManagerFactoryBean entityManagerFactory(DriverManagerDataSource dataSource) { + LocalContainerEntityManagerFactoryBean factory = new LocalContainerEntityManagerFactoryBean(); + factory.setDataSource(dataSource); + factory.setPackagesToScan(AdmissionRow.class.getPackageName()); + factory.setJpaVendorAdapter(new HibernateJpaVendorAdapter()); + factory.setJpaPropertyMap(Map.of("hibernate.hbm2ddl.auto", "create-drop")); + return factory; + } + + @Bean + PlatformTransactionManager transactionManager(jakarta.persistence.EntityManagerFactory factory) { + return new JpaTransactionManager(factory); + } + + @Bean + AdmissionNestedWriter nestedWriter(AdmissionRepository repository) { + return new AdmissionNestedWriter(repository); + } + + @Bean + AdmissionService admissionService( + AdmissionRepository repository, + AdmissionNestedWriter nestedWriter, + MetadataWriteAdmissionCoordinator coordinator) { + return new AdmissionService(repository, nestedWriter, coordinator); + } + } + + @Entity(name = "AdmissionRow") + static class AdmissionRow { + + @Id + @GeneratedValue + private Long id; + + private String label; + + protected AdmissionRow() { + } + + AdmissionRow(String value) { + this.label = value; + } + } + + interface AdmissionRepository extends JpaRepository { + + @Override + S save(S entity); + + @Override + void deleteById(Long id); + + @Query("select row from AdmissionRow row") + @Transactional(readOnly = true) + List declaredReadOnly(); + + @Query("select row from AdmissionRow row") + @Transactional + List declaredWritable(); + } + + static class AdmissionNestedWriter { + + private final AdmissionRepository repository; + + AdmissionNestedWriter(AdmissionRepository repository) { + this.repository = repository; + } + + @Transactional + public void write() { + repository.saveAndFlush(new AdmissionRow("nested")); + } + } + + static class AdmissionService { + + private final AdmissionRepository repository; + private final AdmissionNestedWriter nestedWriter; + private final MetadataWriteAdmissionCoordinator coordinator; + + AdmissionService( + AdmissionRepository repository, + AdmissionNestedWriter nestedWriter, + MetadataWriteAdmissionCoordinator coordinator) { + this.repository = repository; + this.nestedWriter = nestedWriter; + this.coordinator = coordinator; + } + + @Transactional + public void write(String value) { + repository.saveAndFlush(new AdmissionRow(value)); + } + + @Transactional + public void holdWrite( + CountDownLatch admitted, + CountDownLatch finish, + CountDownLatch afterCompletionEntered, + CountDownLatch finishAfterCompletion, + boolean rollback) { + repository.saveAndFlush(new AdmissionRow("held")); + if (afterCompletionEntered != null) { + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCompletion(int status) { + afterCompletionEntered.countDown(); + await(finishAfterCompletion); + } + }); + } + admitted.countDown(); + await(finish); + if (rollback) { + throw new IllegalStateException("rollback requested"); + } + } + + @Transactional + public void outerThenNested(CountDownLatch admitted, CountDownLatch invokeNested) { + repository.saveAndFlush(new AdmissionRow("outer")); + admitted.countDown(); + await(invokeNested); + nestedWriter.write(); + } + + @Transactional(readOnly = true) + public long count() { + return repository.count(); + } + + @Transactional(readOnly = true) + public void readOnlyThenNestedWrite() { + nestedWriter.write(); + } + + @Transactional(readOnly = true) + public void readOnlyOuterHoldingAfterNestedWrite( + CountDownLatch nestedReturned, CountDownLatch finishOuter) { + nestedWriter.write(); + nestedReturned.countDown(); + await(finishOuter); + } + + @Transactional(propagation = Propagation.MANDATORY) + public void mandatoryWrite() { + repository.saveAndFlush(new AdmissionRow("mandatory")); + } + + @Transactional + public MetadataWriteAdmissionSnapshot observeAdmissionInsideTransaction() { + assertThat(TransactionSynchronizationManager.isActualTransactionActive()).isTrue(); + return coordinator.snapshot(); + } + + private static void await(CountDownLatch latch) { + try { + if (!latch.await(2, TimeUnit.SECONDS)) { + throw new IllegalStateException("test latch timed out"); + } + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("test interrupted", exception); + } + } + } +} diff --git a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/transaction/TransactionCompletionPermitRegistryTest.java b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/transaction/TransactionCompletionPermitRegistryTest.java new file mode 100644 index 0000000000..3ce49560a3 --- /dev/null +++ b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/transaction/TransactionCompletionPermitRegistryTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.common.transaction; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.springframework.transaction.support.TransactionSynchronizationManager; + +class TransactionCompletionPermitRegistryTest { + + @AfterEach + void clearTransactionState() { + TransactionSynchronizationManager.clear(); + } + + @Test + void synchronizationRegistrationFailureReleasesPermitAndResource() { + MetadataWriteAdmissionCoordinator coordinator = new MetadataWriteAdmissionCoordinator(); + TransactionCompletionPermitRegistry registry = new TransactionCompletionPermitRegistry(synchronization -> { + throw new IllegalStateException("registration failed"); + }); + TransactionSynchronizationManager.initSynchronization(); + TransactionSynchronizationManager.setActualTransactionActive(true); + MetadataWriteAdmissionCoordinator.TransactionPermit permit = coordinator.admitWritableTransaction(); + + assertThatThrownBy(() -> registry.bind(permit)).isInstanceOf(IllegalStateException.class); + + assertThat(registry.hasPermit()).isFalse(); + assertThat(coordinator.snapshot().activeWritableTransactions()).isZero(); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandler.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandler.java index 7645cad2d7..b89b823c78 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandler.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandler.java @@ -19,9 +19,11 @@ package org.apache.hertzbeat.manager.setup.api; import java.time.Clock; import java.util.Objects; +import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionException; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; import org.apache.hertzbeat.manager.setup.security.SetupUnlockRejected; import org.apache.hertzbeat.manager.setup.workflow.SetupWorkflowConflict; +import org.apache.hertzbeat.manager.support.MetadataWriteMaintenanceErrorResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; @@ -76,6 +78,13 @@ public class SetupExceptionHandler { return response(HttpStatus.BAD_REQUEST, SetupErrorCode.INVALID_REQUEST); } + /** Return the same safe maintenance contract used by ordinary manager endpoints. */ + @ExceptionHandler(MetadataWriteAdmissionException.class) + public ResponseEntity metadataWriteAdmission( + MetadataWriteAdmissionException failure) { + return MetadataWriteMaintenanceErrorResponse.httpResponse(failure); + } + @ExceptionHandler(Exception.class) public ResponseEntity unexpectedFailure(Exception failure) { reportSafely(failure.getClass().getName()); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/support/GlobalExceptionHandler.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/support/GlobalExceptionHandler.java index 77fb782437..84150001e2 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/support/GlobalExceptionHandler.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/support/GlobalExceptionHandler.java @@ -25,6 +25,7 @@ import java.util.Objects; import jakarta.servlet.http.HttpServletResponse; import jakarta.validation.ConstraintViolationException; import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionException; import org.apache.hertzbeat.common.entity.dto.Message; import org.apache.hertzbeat.common.support.exception.CommonException; import org.apache.hertzbeat.alert.notice.AlertNoticeException; @@ -55,6 +56,14 @@ public class GlobalExceptionHandler { private static final String CONNECT_STR = "||"; private static final String UNKNOWN_ERROR_MESSAGE = "unknown error happen"; + /** Return a retryable, cache-safe maintenance response without logging private state. */ + @ExceptionHandler(MetadataWriteAdmissionException.class) + @ResponseBody + ResponseEntity handleMetadataWriteAdmissionException( + MetadataWriteAdmissionException exception) { + return MetadataWriteMaintenanceErrorResponse.httpResponse(exception); + } + /** * Processing probe failure * @param exception Detection anomaly diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/support/MetadataWriteMaintenanceErrorResponse.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/support/MetadataWriteMaintenanceErrorResponse.java new file mode 100644 index 0000000000..7e8ec797d9 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/support/MetadataWriteMaintenanceErrorResponse.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.support; + +import java.util.Objects; +import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionException; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; + +/** Stable, secret-free response for temporarily unavailable metadata writes. */ +public record MetadataWriteMaintenanceErrorResponse(String errorCode, String message) { + + /** Map every shared category to stable HTTP semantics without exposing private state. */ + public static ResponseEntity httpResponse( + MetadataWriteAdmissionException failure) { + Objects.requireNonNull(failure, "failure"); + HttpStatus status = switch (failure.code()) { + case MAINTENANCE_ACTIVE, DRAIN_TIMEOUT, ACQUISITION_INTERRUPTED -> + HttpStatus.SERVICE_UNAVAILABLE; + case OPERATION_CONFLICT -> HttpStatus.CONFLICT; + case INVALID_REQUEST -> HttpStatus.BAD_REQUEST; + }; + MetadataWriteMaintenanceErrorResponse body = new MetadataWriteMaintenanceErrorResponse( + failure.code().wireCode(), failure.safeMessage()); + return ResponseEntity.status(status).header("Cache-Control", "no-store").body(body); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionHttpMappingTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionHttpMappingTest.java new file mode 100644 index 0000000000..6e888fab13 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/common/transaction/MetadataWriteAdmissionHttpMappingTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.common.transaction; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.stream.Stream; +import org.apache.hertzbeat.manager.support.MetadataWriteMaintenanceErrorResponse; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.http.HttpStatus; + +class MetadataWriteAdmissionHttpMappingTest { + + @ParameterizedTest + @MethodSource("admissionFailures") + void mapsEveryAdmissionCategoryToStableSafeHttpSemantics( + MetadataWriteAdmissionException failure, HttpStatus expectedStatus) { + var response = MetadataWriteMaintenanceErrorResponse.httpResponse(failure); + + assertThat(response.getStatusCode()).isEqualTo(expectedStatus); + assertThat(response.getHeaders().getFirst("Cache-Control")).isEqualTo("no-store"); + assertThat(response.getBody()).isEqualTo(new MetadataWriteMaintenanceErrorResponse( + failure.code().wireCode(), failure.safeMessage())); + } + + private static Stream admissionFailures() { + return Stream.of( + Arguments.of(MetadataWriteAdmissionException.metadataWritesPaused(), + HttpStatus.SERVICE_UNAVAILABLE), + Arguments.of(MetadataWriteAdmissionException.operationConflict(), HttpStatus.CONFLICT), + Arguments.of(MetadataWriteAdmissionException.drainTimeout(), HttpStatus.SERVICE_UNAVAILABLE), + Arguments.of(MetadataWriteAdmissionException.acquisitionInterrupted(), + HttpStatus.SERVICE_UNAVAILABLE), + Arguments.of(MetadataWriteAdmissionException.invalidRequest(), HttpStatus.BAD_REQUEST)); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerTest.java index 464d244f11..3a97b5cd79 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerTest.java @@ -35,6 +35,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. import java.io.ByteArrayOutputStream; import java.time.Instant; import java.util.List; +import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionException; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentTopology; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentView; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MaintenanceAdmission; @@ -146,6 +147,19 @@ class DeploymentControllerTest { org.hamcrest.Matchers.containsString("internal_table")))); } + @Test + void metadataWriteAdmissionFailureUsesSafeRetryableEnvelope() throws Exception { + when(workflow.deployment()).thenThrow(MetadataWriteAdmissionException.metadataWritesPaused()); + + mvc.perform(get(DeploymentApiContract.DEPLOYMENT_PATH)) + .andExpect(status().isServiceUnavailable()) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.errorCode").value("metadata_writes_paused")) + .andExpect(jsonPath("$.message").value("Metadata writes are temporarily unavailable")) + .andExpect(content().string(org.hamcrest.Matchers.not( + org.hamcrest.Matchers.containsString("operation-private")))); + } + @Test void missingMigrationPollIsAnExplicitNotFound() throws Exception { when(workflow.migration("missing")).thenReturn(null); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandlerLoggingTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandlerLoggingTest.java index 00d8d5e16f..e925a9ac19 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandlerLoggingTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupExceptionHandlerLoggingTest.java @@ -39,6 +39,7 @@ import java.time.Clock; import java.time.Instant; import java.time.ZoneOffset; import java.util.Arrays; +import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionException; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; import org.apache.hertzbeat.manager.setup.runtime.SetupResponseTransition; import org.apache.hertzbeat.manager.setup.security.SetupHttpUnlockService; @@ -135,4 +136,19 @@ class SetupExceptionHandlerLoggingTest { assertTrue(appender.list.stream().noneMatch(event -> event.getLevel() == Level.ERROR)); } + + @Test + void metadataWriteAdmissionFailureIsSafeServiceUnavailableInsteadOfInternalError() throws Exception { + when(workflow.status()).thenThrow(MetadataWriteAdmissionException.metadataWritesPaused()); + + mvc.perform(get(SetupApiContract.STATUS_PATH)) + .andExpect(status().isServiceUnavailable()) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.errorCode").value("metadata_writes_paused")) + .andExpect(jsonPath("$.message").value("Metadata writes are temporarily unavailable")) + .andExpect(content().string(org.hamcrest.Matchers.not( + org.hamcrest.Matchers.containsString("operation-private")))); + + assertTrue(appender.list.stream().noneMatch(event -> event.getLevel() == Level.ERROR)); + } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/support/GlobalExceptionHandlerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/support/GlobalExceptionHandlerTest.java index 2f72397f8a..144e874ad7 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/support/GlobalExceptionHandlerTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/support/GlobalExceptionHandlerTest.java @@ -11,6 +11,8 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.springframework.http.MediaType.TEXT_EVENT_STREAM_VALUE; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import ch.qos.logback.classic.Level; @@ -18,6 +20,7 @@ import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.read.ListAppender; import jakarta.servlet.http.HttpServletResponse; +import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionException; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.slf4j.LoggerFactory; @@ -125,6 +128,17 @@ class GlobalExceptionHandlerTest { } } + @Test + void metadataWriteAdmissionFailureIsStableNoStoreServiceUnavailable() throws Exception { + mockMvc.perform(MockMvcRequestBuilders.post("/metadata-write-maintenance")) + .andExpect(status().isServiceUnavailable()) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.errorCode").value("metadata_writes_paused")) + .andExpect(jsonPath("$.message").value("Metadata writes are temporarily unavailable")) + .andExpect(content().string(org.hamcrest.Matchers.not( + org.hamcrest.Matchers.containsString("operation-private")))); + } + @RestController private static final class DisconnectController { @@ -149,6 +163,11 @@ class GlobalExceptionHandlerTest { return new SerializationFailureDto(); } + @org.springframework.web.bind.annotation.PostMapping("/metadata-write-maintenance") + void metadataWriteMaintenance() { + throw MetadataWriteAdmissionException.metadataWritesPaused(); + } + private HttpMessageNotWritableException wrappedDisconnectFailure() { return new HttpMessageNotWritableException( PRIVATE_DISCONNECT_DETAIL, diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/MetadataWriteAdmissionStartupContextTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/MetadataWriteAdmissionStartupContextTest.java new file mode 100644 index 0000000000..1791a87509 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/MetadataWriteAdmissionStartupContextTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.lang.reflect.Method; +import java.util.Arrays; +import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionAdvisor; +import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionCoordinator; +import org.apache.hertzbeat.manager.dao.MonitorDao; +import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; +import org.apache.hertzbeat.warehouse.store.DataStorageDispatch; +import org.apache.hertzbeat.warehouse.store.metadata.JdbcMonitorStatusMetadataWriter; +import org.junit.jupiter.api.Test; +import org.springframework.aop.framework.Advised; +import org.springframework.aop.support.AopUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.ApplicationContext; +import org.springframework.data.jpa.repository.support.SimpleJpaRepository; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.bean.override.mockito.MockitoBean; +import org.springframework.transaction.interceptor.BeanFactoryTransactionAttributeSourceAdvisor; +import org.springframework.transaction.interceptor.TransactionAttribute; +import org.springframework.transaction.interceptor.TransactionAttributeSource; +import org.springframework.transaction.interceptor.TransactionInterceptor; + +/** Full application context proof for the metadata write admission advisor chain. */ +@ActiveProfiles("test") +@SpringBootTest(classes = HertzBeatApplication.class, webEnvironment = SpringBootTest.WebEnvironment.NONE) +class MetadataWriteAdmissionStartupContextTest { + + @Autowired + private ApplicationContext context; + + @MockitoBean + private SetupRuntimeTransition setupRuntimeTransition; + + @Test + void startupHasOneAdmissionBoundaryAndOneTransactionSource() throws Exception { + assertThat(context.getBeansOfType(MetadataWriteAdmissionCoordinator.class)).hasSize(1); + assertThat(context.getBeansOfType(MetadataWriteAdmissionAdvisor.class)).hasSize(1); + assertThat(context.getBeansOfType(TransactionAttributeSource.class)).hasSize(1); + MetadataWriteAdmissionAdvisor admissionAdvisor = context.getBean(MetadataWriteAdmissionAdvisor.class); + BeanFactoryTransactionAttributeSourceAdvisor transactionAdvisor = + context.getBean(BeanFactoryTransactionAttributeSourceAdvisor.class); + assertThat(admissionAdvisor.getOrder()).isLessThan(transactionAdvisor.getOrder()); + + JdbcMonitorStatusMetadataWriter writer = context.getBean(JdbcMonitorStatusMetadataWriter.class); + assertThat(AopUtils.isAopProxy(writer)).isTrue(); + assertThat(writer).isInstanceOf(Advised.class); + Advised advised = (Advised) writer; + assertThat(Arrays.stream(advised.getAdvisors()) + .filter(MetadataWriteAdmissionAdvisor.class::isInstance)).hasSize(1); + assertThat(Arrays.stream(advised.getAdvisors()) + .filter(advisor -> advisor.getAdvice() instanceof TransactionInterceptor)).hasSize(1); + assertThat(Arrays.asList(advised.getAdvisors()).indexOf(admissionAdvisor)) + .isLessThan(transactionAdvisorIndex(advised)); + + assertThat(AopUtils.isAopProxy(context.getBean(DataStorageDispatch.class))).isFalse(); + + MonitorDao repository = context.getBean(MonitorDao.class); + assertThat(repository).isInstanceOf(Advised.class); + Advised repositoryProxy = (Advised) repository; + assertThat(repositoryProxy.getTargetSource().getTarget()).isInstanceOf(SimpleJpaRepository.class); + assertThat(AopUtils.isAopProxy(repositoryProxy.getTargetSource().getTarget())).isFalse(); + assertThat(Arrays.stream(repositoryProxy.getAdvisors()) + .filter(MetadataWriteAdmissionAdvisor.class::isInstance)).hasSize(1); + assertThat(Arrays.stream(repositoryProxy.getAdvisors()) + .filter(advisor -> advisor.getAdvice() instanceof TransactionInterceptor)).hasSize(1); + assertThat(admissionAdvisorIndex(repositoryProxy)).isLessThan(transactionAdvisorIndex(repositoryProxy)); + + TransactionInterceptor repositoryTransactionInterceptor = transactionInterceptor(repositoryProxy); + TransactionAttributeSource repositoryAttributes = + repositoryTransactionInterceptor.getTransactionAttributeSource(); + Class repositoryTargetClass = AopUtils.getTargetClass(repositoryProxy.getTargetSource().getTarget()); + assertRepositoryReadOnly(repositoryAttributes, repositoryTargetClass, + MonitorDao.class.getMethod("findAll"), true); + assertRepositoryReadOnly(repositoryAttributes, repositoryTargetClass, + MonitorDao.class.getMethod("save", Object.class), false); + assertRepositoryReadOnly(repositoryAttributes, repositoryTargetClass, + MonitorDao.class.getMethod("deleteById", Object.class), false); + } + + private int admissionAdvisorIndex(Advised advised) { + for (int index = 0; index < advised.getAdvisors().length; index++) { + if (advised.getAdvisors()[index] instanceof MetadataWriteAdmissionAdvisor) { + return index; + } + } + return -1; + } + + private int transactionAdvisorIndex(Advised advised) { + for (int index = 0; index < advised.getAdvisors().length; index++) { + if (advised.getAdvisors()[index].getAdvice() instanceof TransactionInterceptor) { + return index; + } + } + return -1; + } + + private TransactionInterceptor transactionInterceptor(Advised advised) { + return (TransactionInterceptor) advised.getAdvisors()[transactionAdvisorIndex(advised)].getAdvice(); + } + + private void assertRepositoryReadOnly( + TransactionAttributeSource attributes, + Class targetClass, + Method interfaceMethod, + boolean expectedReadOnly) { + TransactionAttribute attribute = attributes.getTransactionAttribute(interfaceMethod, targetClass); + assertThat(attribute).isNotNull(); + assertThat(attribute.isReadOnly()).isEqualTo(expectedReadOnly); + } +} diff --git a/hertzbeat-warehouse/pom.xml b/hertzbeat-warehouse/pom.xml index e65b30d536..90804b7d70 100644 --- a/hertzbeat-warehouse/pom.xml +++ b/hertzbeat-warehouse/pom.xml @@ -163,6 +163,11 @@ org.apache.arrow arrow-memory-netty + + com.h2database + h2 + test + org.xerial.snappy diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatch.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatch.java index 9563e7ab4e..f39933f2ba 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatch.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatch.java @@ -17,25 +17,23 @@ package org.apache.hertzbeat.warehouse.store; -import jakarta.persistence.EntityManager; -import jakarta.persistence.PersistenceContext; import java.util.List; import java.util.Objects; import lombok.extern.slf4j.Slf4j; -import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.log.LogEntry; -import org.apache.hertzbeat.common.entity.manager.Monitor; import org.apache.hertzbeat.common.entity.message.CollectRep; import org.apache.hertzbeat.common.queue.CommonDataQueue; import org.apache.hertzbeat.common.support.exception.CommonDataQueueUnknownException; +import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionException; import org.apache.hertzbeat.common.util.BackoffUtils; import org.apache.hertzbeat.common.util.ExponentialBackoff; import org.apache.hertzbeat.plugin.PostCollectPlugin; import org.apache.hertzbeat.plugin.runner.PluginRunner; import org.apache.hertzbeat.warehouse.WarehouseWorkerPool; import org.apache.hertzbeat.warehouse.store.history.tsdb.HistoryDataWriter; +import org.apache.hertzbeat.warehouse.store.metadata.MonitorAvailability; +import org.apache.hertzbeat.warehouse.store.metadata.MonitorStatusMetadataWriter; import org.apache.hertzbeat.warehouse.store.realtime.RealTimeDataWriter; -import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; /** @@ -47,23 +45,21 @@ public class DataStorageDispatch { private final CommonDataQueue commonDataQueue; private final WarehouseWorkerPool workerPool; - private final JdbcTemplate jdbcTemplate; + private final MonitorStatusMetadataWriter monitorStatusWriter; private final RealTimeDataWriter realTimeDataWriter; private final List historyDataWriters; private final PluginRunner pluginRunner; private static final int LOG_BATCH_SIZE = 1000; - @PersistenceContext - private EntityManager entityManager; public DataStorageDispatch(CommonDataQueue commonDataQueue, WarehouseWorkerPool workerPool, - JdbcTemplate jdbcTemplate, + MonitorStatusMetadataWriter monitorStatusWriter, List historyDataWriters, RealTimeDataWriter realTimeDataWriter, PluginRunner pluginRunner) { this.commonDataQueue = commonDataQueue; this.workerPool = workerPool; - this.jdbcTemplate = jdbcTemplate; + this.monitorStatusWriter = monitorStatusWriter; this.realTimeDataWriter = realTimeDataWriter; this.historyDataWriters = historyDataWriters == null ? List.of() : historyDataWriters.stream().filter(Objects::nonNull).toList(); @@ -83,16 +79,7 @@ public class DataStorageDispatch { continue; } backoff.reset(); - try { - calculateMonitorStatus(metricsData); - HistoryDataWriter historyDataWriter = resolveMetricsHistoryWriter(); - if (historyDataWriter != null) { - historyDataWriter.saveData(metricsData); - } - pluginRunner.pluginExecute(PostCollectPlugin.class, ((postCollectPlugin, pluginContext) -> postCollectPlugin.execute(metricsData, pluginContext))); - } finally { - realTimeDataWriter.saveData(metricsData); - } + persistMetricsData(metricsData); } catch (InterruptedException interruptedException) { Thread.currentThread().interrupt(); } catch (CommonDataQueueUnknownException ue) { @@ -159,19 +146,28 @@ public class DataStorageDispatch { long id = metricsData.getId(); CollectRep.Code code = metricsData.getCode(); try { - String sql = "UPDATE hzb_monitor SET status = ? WHERE id = ? AND status <> ? AND status <> ?"; - byte status = code == CollectRep.Code.SUCCESS - ? CommonConstants.MONITOR_UP_CODE - : CommonConstants.MONITOR_DOWN_CODE; - // Paused monitors must remain paused. Every other non-current - // state, including Pending, converges on the first priority-0 result. - int matchedRows = jdbcTemplate.update(sql, status, id, CommonConstants.MONITOR_PAUSED_CODE, status); - if (matchedRows > 0) { - entityManager.getEntityManagerFactory().getCache().evict(Monitor.class, id); - } + MonitorAvailability availability = code == CollectRep.Code.SUCCESS + ? MonitorAvailability.UP : MonitorAvailability.DOWN; + monitorStatusWriter.updateAvailability(id, availability); + } catch (MetadataWriteAdmissionException exception) { + log.debug("Monitor status metadata write skipped during maintenance"); } catch (Exception e) { log.error("Update monitor status failed for monitor id: {}", id, e); } } } + + protected void persistMetricsData(CollectRep.MetricsData metricsData) { + try { + calculateMonitorStatus(metricsData); + HistoryDataWriter historyDataWriter = resolveMetricsHistoryWriter(); + if (historyDataWriter != null) { + historyDataWriter.saveData(metricsData); + } + pluginRunner.pluginExecute(PostCollectPlugin.class, + (postCollectPlugin, pluginContext) -> postCollectPlugin.execute(metricsData, pluginContext)); + } finally { + realTimeDataWriter.saveData(metricsData); + } + } } diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/metadata/JdbcMonitorStatusMetadataWriter.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/metadata/JdbcMonitorStatusMetadataWriter.java new file mode 100644 index 0000000000..06e27f89ac --- /dev/null +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/metadata/JdbcMonitorStatusMetadataWriter.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.warehouse.store.metadata; + +import jakarta.persistence.EntityManagerFactory; +import java.util.Objects; +import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.entity.manager.Monitor; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +/** Transactional JDBC adapter for the monitor-status metadata port. */ +@Component +public class JdbcMonitorStatusMetadataWriter implements MonitorStatusMetadataWriter { + + private static final String UPDATE_STATUS = + "UPDATE hzb_monitor SET status = ? WHERE id = ? AND status <> ? AND status <> ?"; + + private final JdbcTemplate jdbcTemplate; + private final EntityManagerFactory entityManagerFactory; + + public JdbcMonitorStatusMetadataWriter(JdbcTemplate jdbcTemplate, EntityManagerFactory entityManagerFactory) { + this.jdbcTemplate = jdbcTemplate; + this.entityManagerFactory = entityManagerFactory; + } + + @Override + @Transactional + public void updateAvailability(long monitorId, MonitorAvailability availability) { + Objects.requireNonNull(availability, "availability"); + byte status = switch (availability) { + case UP -> CommonConstants.MONITOR_UP_CODE; + case DOWN -> CommonConstants.MONITOR_DOWN_CODE; + }; + int matchedRows = jdbcTemplate.update(UPDATE_STATUS, status, monitorId, + CommonConstants.MONITOR_PAUSED_CODE, status); + if (matchedRows > 0) { + entityManagerFactory.getCache().evict(Monitor.class, monitorId); + } + } +} diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/metadata/MonitorAvailability.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/metadata/MonitorAvailability.java new file mode 100644 index 0000000000..43d126ccb8 --- /dev/null +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/metadata/MonitorAvailability.java @@ -0,0 +1,14 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.warehouse.store.metadata; + +/** Availability outcome derived from a priority-zero collection result. */ +public enum MonitorAvailability { + UP, + DOWN +} diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/metadata/MonitorStatusMetadataWriter.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/metadata/MonitorStatusMetadataWriter.java new file mode 100644 index 0000000000..c2128dc2cc --- /dev/null +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/metadata/MonitorStatusMetadataWriter.java @@ -0,0 +1,15 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.warehouse.store.metadata; + +/** Typed metadata write port for monitor availability convergence. */ +public interface MonitorStatusMetadataWriter { + + /** Converge a non-paused monitor to the supplied availability. */ + void updateAvailability(long monitorId, MonitorAvailability availability); +} diff --git a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatchContextTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatchContextTest.java index 95054ec33e..983a72ca39 100644 --- a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatchContextTest.java +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatchContextTest.java @@ -20,25 +20,23 @@ package org.apache.hertzbeat.warehouse.store; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; -import jakarta.persistence.EntityManagerFactory; import org.apache.hertzbeat.common.queue.CommonDataQueue; import org.apache.hertzbeat.plugin.runner.PluginRunner; import org.apache.hertzbeat.warehouse.WarehouseWorkerPool; import org.apache.hertzbeat.warehouse.store.history.tsdb.HistoryDataWriter; +import org.apache.hertzbeat.warehouse.store.metadata.MonitorStatusMetadataWriter; import org.apache.hertzbeat.warehouse.store.realtime.RealTimeDataWriter; import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.runner.ApplicationContextRunner; -import org.springframework.jdbc.core.JdbcTemplate; class DataStorageDispatchContextTest { private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() .withBean(CommonDataQueue.class, () -> mock(CommonDataQueue.class)) .withBean(WarehouseWorkerPool.class, () -> mock(WarehouseWorkerPool.class)) - .withBean(JdbcTemplate.class, () -> mock(JdbcTemplate.class)) + .withBean(MonitorStatusMetadataWriter.class, () -> mock(MonitorStatusMetadataWriter.class)) .withBean(RealTimeDataWriter.class, () -> mock(RealTimeDataWriter.class)) .withBean(PluginRunner.class, () -> mock(PluginRunner.class)) - .withBean(EntityManagerFactory.class, () -> mock(EntityManagerFactory.class)) .withBean("duckdbDatabaseDataStorage", HistoryDataWriter.class, () -> mock(HistoryDataWriter.class)) .withBean("greptimeDbDataStorage", HistoryDataWriter.class, () -> mock(HistoryDataWriter.class)) .withBean(DataStorageDispatch.class); diff --git a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatchMaintenanceContinuityTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatchMaintenanceContinuityTest.java new file mode 100644 index 0000000000..50fe6bc14a --- /dev/null +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatchMaintenanceContinuityTest.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.warehouse.store; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +import java.util.List; +import java.util.function.BiConsumer; +import org.apache.hertzbeat.common.entity.plugin.PluginContext; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.queue.CommonDataQueue; +import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionException; +import org.apache.hertzbeat.plugin.PostCollectPlugin; +import org.apache.hertzbeat.plugin.runner.PluginRunner; +import org.apache.hertzbeat.warehouse.WarehouseWorkerPool; +import org.apache.hertzbeat.warehouse.store.history.tsdb.HistoryDataWriter; +import org.apache.hertzbeat.warehouse.store.metadata.MonitorAvailability; +import org.apache.hertzbeat.warehouse.store.metadata.MonitorStatusMetadataWriter; +import org.apache.hertzbeat.warehouse.store.realtime.RealTimeDataWriter; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InOrder; +import org.springframework.boot.test.system.CapturedOutput; +import org.springframework.boot.test.system.OutputCaptureExtension; + +@ExtendWith(OutputCaptureExtension.class) +class DataStorageDispatchMaintenanceContinuityTest { + + @Test + void maintenanceMetadataRejectionDoesNotInterruptTelemetryPersistenceOrder(CapturedOutput output) { + MonitorStatusMetadataWriter statusWriter = mock(MonitorStatusMetadataWriter.class); + HistoryDataWriter historyWriter = mock(HistoryDataWriter.class); + RealTimeDataWriter realTimeWriter = mock(RealTimeDataWriter.class); + PluginRunner pluginRunner = mock(PluginRunner.class); + MetadataWriteAdmissionException rejection = MetadataWriteAdmissionException.metadataWritesPaused(); + doThrow(rejection).when(statusWriter).updateAvailability(anyLong(), any()); + DataStorageDispatch dispatch = new DataStorageDispatch( + mock(CommonDataQueue.class), + mock(WarehouseWorkerPool.class), + statusWriter, + List.of(historyWriter), + realTimeWriter, + pluginRunner); + CollectRep.MetricsData metrics = CollectRep.MetricsData.newBuilder() + .setId(42L) + .setPriority(0) + .setCode(CollectRep.Code.SUCCESS) + .build(); + + dispatch.persistMetricsData(metrics); + + InOrder order = inOrder(statusWriter, historyWriter, pluginRunner, realTimeWriter); + order.verify(statusWriter).updateAvailability(42L, MonitorAvailability.UP); + order.verify(historyWriter).saveData(metrics); + order.verify(pluginRunner).pluginExecute(eq(PostCollectPlugin.class), pluginExecution()); + order.verify(realTimeWriter).saveData(metrics); + assertThat(output).doesNotContain("ERROR").doesNotContain("MetadataWriteAdmissionException"); + } + + @Test + void ordinaryMetadataFailureAlsoLeavesTelemetryPipelineRunning() { + MonitorStatusMetadataWriter statusWriter = mock(MonitorStatusMetadataWriter.class); + HistoryDataWriter historyWriter = mock(HistoryDataWriter.class); + RealTimeDataWriter realTimeWriter = mock(RealTimeDataWriter.class); + PluginRunner pluginRunner = mock(PluginRunner.class); + doThrow(new IllegalStateException("metadata unavailable")) + .when(statusWriter).updateAvailability(anyLong(), any()); + DataStorageDispatch dispatch = dispatch(statusWriter, historyWriter, realTimeWriter, pluginRunner); + CollectRep.MetricsData metrics = metrics(); + + dispatch.persistMetricsData(metrics); + + InOrder order = inOrder(statusWriter, historyWriter, pluginRunner, realTimeWriter); + order.verify(statusWriter).updateAvailability(42L, MonitorAvailability.UP); + order.verify(historyWriter).saveData(metrics); + order.verify(pluginRunner).pluginExecute(eq(PostCollectPlugin.class), pluginExecution()); + order.verify(realTimeWriter).saveData(metrics); + } + + @Test + void historyFailureSkipsPluginButStillUpdatesRealtime() { + MonitorStatusMetadataWriter statusWriter = mock(MonitorStatusMetadataWriter.class); + HistoryDataWriter historyWriter = mock(HistoryDataWriter.class); + RealTimeDataWriter realTimeWriter = mock(RealTimeDataWriter.class); + PluginRunner pluginRunner = mock(PluginRunner.class); + doThrow(new IllegalStateException("history unavailable")).when(historyWriter).saveData(any()); + DataStorageDispatch dispatch = dispatch(statusWriter, historyWriter, realTimeWriter, pluginRunner); + CollectRep.MetricsData metrics = metrics(); + + assertThatThrownBy(() -> dispatch.persistMetricsData(metrics)) + .isInstanceOf(IllegalStateException.class); + + verify(statusWriter).updateAvailability(42L, MonitorAvailability.UP); + verify(historyWriter).saveData(metrics); + verify(pluginRunner, never()).pluginExecute(eq(PostCollectPlugin.class), pluginExecution()); + verify(realTimeWriter).saveData(metrics); + } + + private DataStorageDispatch dispatch( + MonitorStatusMetadataWriter statusWriter, + HistoryDataWriter historyWriter, + RealTimeDataWriter realTimeWriter, + PluginRunner pluginRunner) { + return new DataStorageDispatch( + mock(CommonDataQueue.class), mock(WarehouseWorkerPool.class), statusWriter, + List.of(historyWriter), realTimeWriter, pluginRunner); + } + + private CollectRep.MetricsData metrics() { + return CollectRep.MetricsData.newBuilder() + .setId(42L) + .setPriority(0) + .setCode(CollectRep.Code.SUCCESS) + .build(); + } + + @SuppressWarnings("unchecked") + private BiConsumer pluginExecution() { + return any(BiConsumer.class); + } +} diff --git a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatchStatusTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatchStatusTest.java index 2b85e5a637..74ba6d68ed 100644 --- a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatchStatusTest.java +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatchStatusTest.java @@ -21,24 +21,24 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import java.util.List; -import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.message.CollectRep; import org.apache.hertzbeat.common.queue.CommonDataQueue; import org.apache.hertzbeat.plugin.runner.PluginRunner; import org.apache.hertzbeat.warehouse.WarehouseWorkerPool; +import org.apache.hertzbeat.warehouse.store.metadata.MonitorAvailability; +import org.apache.hertzbeat.warehouse.store.metadata.MonitorStatusMetadataWriter; import org.apache.hertzbeat.warehouse.store.realtime.RealTimeDataWriter; import org.junit.jupiter.api.Test; -import org.springframework.jdbc.core.JdbcTemplate; class DataStorageDispatchStatusTest { @Test void firstAvailabilityResultCanReplacePendingStatus() { - JdbcTemplate jdbcTemplate = mock(JdbcTemplate.class); + MonitorStatusMetadataWriter statusWriter = mock(MonitorStatusMetadataWriter.class); DataStorageDispatch dispatch = new DataStorageDispatch( mock(CommonDataQueue.class), mock(WarehouseWorkerPool.class), - jdbcTemplate, + statusWriter, List.of(), mock(RealTimeDataWriter.class), mock(PluginRunner.class)); @@ -50,11 +50,6 @@ class DataStorageDispatchStatusTest { dispatch.calculateMonitorStatus(firstResult); - verify(jdbcTemplate).update( - "UPDATE hzb_monitor SET status = ? WHERE id = ? AND status <> ? AND status <> ?", - CommonConstants.MONITOR_UP_CODE, - 42L, - CommonConstants.MONITOR_PAUSED_CODE, - CommonConstants.MONITOR_UP_CODE); + verify(statusWriter).updateAvailability(42L, MonitorAvailability.UP); } } diff --git a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/metadata/MonitorStatusMetadataWriterIntegrationTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/metadata/MonitorStatusMetadataWriterIntegrationTest.java new file mode 100644 index 0000000000..5be543cf4e --- /dev/null +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/metadata/MonitorStatusMetadataWriterIntegrationTest.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.warehouse.store.metadata; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; + +import jakarta.persistence.EntityManagerFactory; +import java.time.Duration; +import javax.sql.DataSource; +import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.entity.manager.Monitor; +import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionConfiguration; +import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionCoordinator; +import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionException; +import org.apache.hertzbeat.common.transaction.MetadataWriteMaintenanceLease; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.jdbc.datasource.DriverManagerDataSource; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = MonitorStatusMetadataWriterIntegrationTest.TestConfiguration.class) +class MonitorStatusMetadataWriterIntegrationTest { + + @Autowired + private MonitorStatusMetadataWriter writer; + + @Autowired + private MetadataWriteAdmissionCoordinator coordinator; + + @Autowired + private JdbcTemplate jdbcTemplate; + + @Autowired + private EntityManagerFactory entityManagerFactory; + + @BeforeEach + void seedMonitor() { + jdbcTemplate.execute("CREATE TABLE IF NOT EXISTS hzb_monitor (id BIGINT PRIMARY KEY, status TINYINT)"); + jdbcTemplate.update("DELETE FROM hzb_monitor"); + jdbcTemplate.update("INSERT INTO hzb_monitor (id, status) VALUES (?, ?)", + 42L, CommonConstants.MONITOR_PENDING_CODE); + clearInvocations(entityManagerFactory.getCache()); + } + + @Test + void typedWriterPreservesPausedCurrentAndCacheEvictionSemantics() { + writer.updateAvailability(42L, MonitorAvailability.UP); + assertThat(status()).isEqualTo(CommonConstants.MONITOR_UP_CODE); + verify(entityManagerFactory.getCache()).evict(Monitor.class, 42L); + + writer.updateAvailability(42L, MonitorAvailability.UP); + jdbcTemplate.update("UPDATE hzb_monitor SET status = ? WHERE id = ?", + CommonConstants.MONITOR_PAUSED_CODE, 42L); + writer.updateAvailability(42L, MonitorAvailability.DOWN); + + assertThat(status()).isEqualTo(CommonConstants.MONITOR_PAUSED_CODE); + verifyNoMoreInteractions(entityManagerFactory.getCache()); + } + + @Test + void maintenanceGateRejectsTypedMetadataWriteAndReleaseRestoresIt() { + try (MetadataWriteMaintenanceLease ignored = coordinator.acquire( + "monitor-status-maintenance", Duration.ofSeconds(1))) { + assertThatThrownBy(() -> writer.updateAvailability(42L, MonitorAvailability.DOWN)) + .isInstanceOf(MetadataWriteAdmissionException.class); + assertThat(status()).isEqualTo(CommonConstants.MONITOR_PENDING_CODE); + } + + writer.updateAvailability(42L, MonitorAvailability.DOWN); + assertThat(status()).isEqualTo(CommonConstants.MONITOR_DOWN_CODE); + } + + private byte status() { + return jdbcTemplate.queryForObject( + "SELECT status FROM hzb_monitor WHERE id = ?", Byte.class, 42L); + } + + @Configuration(proxyBeanMethods = false) + @EnableTransactionManagement(order = 200) + @Import({MetadataWriteAdmissionConfiguration.class, JdbcMonitorStatusMetadataWriter.class}) + static class TestConfiguration { + + @Bean + DataSource dataSource() { + return new DriverManagerDataSource("jdbc:h2:mem:monitor-status;DB_CLOSE_DELAY=-1", "sa", ""); + } + + @Bean + JdbcTemplate jdbcTemplate(DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + @Bean + PlatformTransactionManager transactionManager(DataSource dataSource) { + return new DataSourceTransactionManager(dataSource); + } + + @Bean + EntityManagerFactory entityManagerFactory() { + return mock(EntityManagerFactory.class, RETURNS_DEEP_STUBS); + } + } +} From 2d2a423c7974bf7c54c2e4b0063058de66abeecb Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 00:45:40 +0800 Subject: [PATCH 36/71] Quiesce metadata control-plane writers --- .../sd/ServiceDiscoveryMaintenanceGate.java | 170 ++++++++ .../component/sd/ServiceDiscoveryWorker.java | 87 +++- .../component/status/CalculateStatus.java | 194 +++++---- .../status/PausableDispatchTask.java | 158 ++++++++ .../maintenance/MaintenanceDeadline.java | 56 +++ .../MetadataMaintenanceCoordinator.java | 193 +++++++++ .../MetadataMaintenanceErrorCode.java | 29 ++ .../MetadataMaintenanceException.java | 64 +++ .../maintenance/MetadataMaintenanceLease.java | 39 ++ .../MetadataMaintenanceParticipant.java | 22 + .../maintenance/MetadataMaintenancePhase.java | 15 + .../MetadataMaintenanceSnapshot.java | 13 + .../sd/ServiceDiscoveryWorkerTest.java | 189 ++++++++- .../component/status/CalculateStatusTest.java | 138 +++++++ .../status/PausableDispatchTaskTest.java | 89 +++++ .../maintenance/MaintenanceDeadlineTest.java | 39 ++ .../MetadataMaintenanceCoordinatorTest.java | 378 ++++++++++++++++++ ...adataWriteAdmissionStartupContextTest.java | 14 + 18 files changed, 1771 insertions(+), 116 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryMaintenanceGate.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/PausableDispatchTask.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MaintenanceDeadline.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceCoordinator.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceErrorCode.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceException.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceLease.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceParticipant.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenancePhase.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceSnapshot.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/status/PausableDispatchTaskTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/MaintenanceDeadlineTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceCoordinatorTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryMaintenanceGate.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryMaintenanceGate.java new file mode 100644 index 0000000000..f7e550c1c8 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryMaintenanceGate.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.component.sd; + +import java.time.Duration; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; +import org.apache.hertzbeat.manager.maintenance.MaintenanceDeadline; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceException; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenancePhase; + +/** Admission and drain state for the single service-discovery consumer loop. */ +final class ServiceDiscoveryMaintenanceGate { + + private final ReentrantLock lock = new ReentrantLock(); + private final Condition stateChanged = lock.newCondition(); + private MetadataMaintenancePhase phase = MetadataMaintenancePhase.RUNNING; + private Thread pollingThread; + private boolean polling; + private boolean processing; + private boolean maintenanceWakeup; + private boolean terminal; + + void beforePoll() throws InterruptedException { + lock.lockInterruptibly(); + try { + while (phase != MetadataMaintenancePhase.RUNNING && !terminal) { + stateChanged.await(); + } + if (terminal) { + throw new InterruptedException(); + } + polling = true; + pollingThread = Thread.currentThread(); + } finally { + lock.unlock(); + } + } + + /** Finish acquisition and promote a returned message to in-flight work. */ + boolean pollCompleted(boolean messageReturned) { + lock.lock(); + try { + polling = false; + pollingThread = null; + if (messageReturned) { + processing = true; + } + boolean clearMaintenanceInterrupt = maintenanceWakeup && !terminal; + maintenanceWakeup = false; + stateChanged.signalAll(); + return clearMaintenanceInterrupt; + } finally { + lock.unlock(); + } + } + + boolean pollInterrupted() { + lock.lock(); + try { + polling = false; + pollingThread = null; + boolean expected = maintenanceWakeup && !terminal; + maintenanceWakeup = false; + stateChanged.signalAll(); + return expected; + } finally { + lock.unlock(); + } + } + + void workCompleted() { + lock.lock(); + try { + processing = false; + stateChanged.signalAll(); + } finally { + lock.unlock(); + } + } + + void quiesce(Duration timeout) { + MaintenanceDeadline deadline = MaintenanceDeadline.start(timeout); + try { + lock.lockInterruptibly(); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw MetadataMaintenanceException.quiesceInterrupted(); + } + try { + if (phase == MetadataMaintenancePhase.QUIESCED) { + return; + } + if (phase == MetadataMaintenancePhase.RUNNING) { + phase = MetadataMaintenancePhase.QUIESCING; + if (pollingThread != null) { + maintenanceWakeup = true; + pollingThread.interrupt(); + } + } + while (polling || processing) { + long remainingNanos = deadline.remainingNanos(); + if (remainingNanos <= 0) { + reopen(); + throw MetadataMaintenanceException.quiesceTimeout(); + } + try { + stateChanged.awaitNanos(remainingNanos); + } catch (InterruptedException exception) { + reopen(); + Thread.currentThread().interrupt(); + throw MetadataMaintenanceException.quiesceInterrupted(); + } + } + phase = MetadataMaintenancePhase.QUIESCED; + stateChanged.signalAll(); + } finally { + lock.unlock(); + } + } + + void resume() { + lock.lock(); + try { + if (terminal) { + return; + } + if (phase != MetadataMaintenancePhase.RUNNING) { + reopen(); + } + } finally { + lock.unlock(); + } + } + + void stop() { + lock.lock(); + try { + if (terminal) { + return; + } + terminal = true; + if (pollingThread != null) { + pollingThread.interrupt(); + } + stateChanged.signalAll(); + } finally { + lock.unlock(); + } + } + + MetadataMaintenancePhase phase() { + lock.lock(); + try { + return phase; + } finally { + lock.unlock(); + } + } + + private void reopen() { + phase = MetadataMaintenancePhase.RUNNING; + stateChanged.signalAll(); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryWorker.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryWorker.java index e214124313..181e3690a2 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryWorker.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryWorker.java @@ -18,6 +18,14 @@ package org.apache.hertzbeat.manager.component.sd; import com.google.common.collect.Maps; +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.arrow.RowWrapper; @@ -35,26 +43,24 @@ import org.apache.hertzbeat.manager.dao.CollectorMonitorBindDao; import org.apache.hertzbeat.manager.dao.MonitorBindDao; import org.apache.hertzbeat.manager.dao.MonitorDao; import org.apache.hertzbeat.manager.dao.ParamDao; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceParticipant; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenancePhase; import org.apache.hertzbeat.manager.scheduler.ManagerWorkerPool; import org.apache.hertzbeat.manager.service.MonitorService; +import org.springframework.beans.factory.DisposableBean; import org.springframework.beans.factory.InitializingBean; +import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; -import java.time.LocalDateTime; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - /** * Service Discovery Worker */ @Slf4j @Component @ConditionalOnNormalBusinessRuntime -public class ServiceDiscoveryWorker implements InitializingBean { +@Order(100) +public class ServiceDiscoveryWorker + implements InitializingBean, DisposableBean, MetadataMaintenanceParticipant { private static final String FILED_HOST = "host"; private static final String FILED_PORT = "port"; @@ -65,6 +71,7 @@ public class ServiceDiscoveryWorker implements InitializingBean { private final CollectorMonitorBindDao collectorMonitorBindDao; private final CommonDataQueue dataQueue; private final ManagerWorkerPool workerPool; + private final ServiceDiscoveryMaintenanceGate maintenanceGate = new ServiceDiscoveryMaintenanceGate(); public ServiceDiscoveryWorker(MonitorService monitorService, ParamDao paramDao, MonitorDao monitorDao, MonitorBindDao monitorBindDao, CollectorMonitorBindDao collectorMonitorBindDao, @@ -83,12 +90,65 @@ public class ServiceDiscoveryWorker implements InitializingBean { workerPool.executeLongRunning(new SdUpdateTask()); } + @Override + public String participantId() { + return "service-discovery"; + } + + @Override + public void quiesce(Duration timeout) { + maintenanceGate.quiesce(timeout); + } + + @Override + public void resume() { + maintenanceGate.resume(); + } + + MetadataMaintenancePhase maintenancePhase() { + return maintenanceGate.phase(); + } + + @Override + public void destroy() { + maintenanceGate.stop(); + } + private class SdUpdateTask implements Runnable { @Override public void run() { ExponentialBackoff backoff = new ExponentialBackoff(50L, 1000L); while (!Thread.currentThread().isInterrupted()) { - try (final CollectRep.MetricsData metricsData = dataQueue.pollServiceDiscoveryData()) { + CollectRep.MetricsData polledData; + try { + maintenanceGate.beforePoll(); + polledData = dataQueue.pollServiceDiscoveryData(); + } catch (InterruptedException interruptedException) { + if (maintenanceGate.pollInterrupted()) { + Thread.interrupted(); + continue; + } + Thread.currentThread().interrupt(); + break; + } catch (RuntimeException exception) { + boolean clearMaintenanceInterrupt = maintenanceGate.pollCompleted(false); + if (clearMaintenanceInterrupt) { + Thread.interrupted(); + } + if (exception instanceof CommonDataQueueUnknownException) { + if (!BackoffUtils.shouldContinueAfterBackoff(backoff)) { + break; + } + } else { + log.error(exception.getMessage(), exception); + } + continue; + } + boolean clearMaintenanceInterrupt = maintenanceGate.pollCompleted(polledData != null); + if (clearMaintenanceInterrupt) { + Thread.interrupted(); + } + try (final CollectRep.MetricsData metricsData = polledData) { if (metricsData == null) { continue; } @@ -165,15 +225,16 @@ public class ServiceDiscoveryWorker implements InitializingBean { final Set needCancelMonitorIdSet = subMonitorBindMap.values().stream() .map(MonitorBind::getMonitorId).collect(Collectors.toSet()); monitorService.deleteMonitors(needCancelMonitorIdSet); - } catch (InterruptedException interruptedException) { - Thread.currentThread().interrupt(); - break; } catch (CommonDataQueueUnknownException ue) { if (!BackoffUtils.shouldContinueAfterBackoff(backoff)) { break; } } catch (Exception exception) { log.error(exception.getMessage(), exception); + } finally { + if (polledData != null) { + maintenanceGate.workCompleted(); + } } } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatus.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatus.java index 4831c9ee23..528bfd7153 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatus.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatus.java @@ -47,8 +47,13 @@ import org.apache.hertzbeat.manager.dao.MonitorDao; import org.apache.hertzbeat.manager.dao.StatusPageComponentDao; import org.apache.hertzbeat.manager.dao.StatusPageHistoryDao; import org.apache.hertzbeat.manager.dao.StatusPageOrgDao; +import org.apache.hertzbeat.manager.maintenance.MaintenanceDeadline; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceException; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceParticipant; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenancePhase; import org.springframework.beans.factory.DisposableBean; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.annotation.Order; import org.springframework.data.jpa.domain.Specification; import org.springframework.stereotype.Component; @@ -57,7 +62,8 @@ import org.springframework.stereotype.Component; */ @Component @Slf4j -public class CalculateStatus implements DisposableBean { +@Order(200) +public class CalculateStatus implements DisposableBean, MetadataMaintenanceParticipant { private static final int DEFAULT_CALCULATE_INTERVAL_TIME = 300; @@ -80,12 +86,14 @@ public class CalculateStatus implements DisposableBean { private ExecutorService combineHistoryExecutor; - private ScheduledDispatchTask calculateTask; + private PausableDispatchTask calculateTask; - private ScheduledDispatchTask combineHistoryTask; + private PausableDispatchTask combineHistoryTask; private boolean started; + private volatile MetadataMaintenancePhase maintenancePhase = MetadataMaintenancePhase.RUNNING; + public CalculateStatus(StatusPageOrgDao statusPageOrgDao, StatusPageComponentDao statusPageComponentDao, StatusProperties statusProperties, StatusPageHistoryDao statusPageHistoryDao, MonitorDao monitorDao) { @@ -121,10 +129,10 @@ public class CalculateStatus implements DisposableBean { "Status calculate worker has uncaughtException."); combineHistoryExecutor = createVirtualExecutor(virtualThreadProperties, "status-page-history-vt-", "History combine worker has uncaughtException."); - ScheduledDispatchTask currentCalculateTask = - new ScheduledDispatchTask(calculateExecutor, this::runCalculate); - ScheduledDispatchTask currentCombineHistoryTask = - new ScheduledDispatchTask(combineHistoryExecutor, this::runCombineHistory); + PausableDispatchTask currentCalculateTask = + new PausableDispatchTask(calculateExecutor, this::runCalculate); + PausableDispatchTask currentCombineHistoryTask = + new PausableDispatchTask(combineHistoryExecutor, this::runCombineHistory); calculateTask = currentCalculateTask; combineHistoryTask = currentCombineHistoryTask; startCalculate(currentCalculateTask); @@ -140,11 +148,73 @@ public class CalculateStatus implements DisposableBean { return started; } - private void startCalculate(ScheduledDispatchTask currentCalculateTask) { + @Override + public String participantId() { + return "status-calculation"; + } + + @Override + public void quiesce(Duration timeout) { + MaintenanceDeadline deadline = MaintenanceDeadline.start(timeout); + PausableDispatchTask currentCalculateTask; + PausableDispatchTask currentCombineHistoryTask; + synchronized (this) { + if (maintenancePhase == MetadataMaintenancePhase.QUIESCED) { + return; + } + maintenancePhase = MetadataMaintenancePhase.QUIESCING; + currentCalculateTask = calculateTask; + currentCombineHistoryTask = combineHistoryTask; + if (currentCalculateTask != null) { + currentCalculateTask.pauseAdmission(); + } + if (currentCombineHistoryTask != null) { + currentCombineHistoryTask.pauseAdmission(); + } + } + try { + if (currentCalculateTask != null) { + currentCalculateTask.awaitDrained(deadline); + } + if (currentCombineHistoryTask != null) { + currentCombineHistoryTask.awaitDrained(deadline); + } + maintenancePhase = MetadataMaintenancePhase.QUIESCED; + } catch (MetadataMaintenanceException exception) { + resumeTasks(currentCalculateTask, currentCombineHistoryTask); + maintenancePhase = MetadataMaintenancePhase.RUNNING; + throw exception; + } + } + + @Override + public synchronized void resume() { + if (maintenancePhase == MetadataMaintenancePhase.RUNNING) { + return; + } + maintenancePhase = MetadataMaintenancePhase.RUNNING; + resumeTasks(calculateTask, combineHistoryTask); + } + + MetadataMaintenancePhase maintenancePhase() { + return maintenancePhase; + } + + private void resumeTasks(PausableDispatchTask first, PausableDispatchTask second) { + if (second != null) { + second.resumeAdmission(); + } + if (first != null) { + first.resumeAdmission(); + } + } + + + private void startCalculate(PausableDispatchTask currentCalculateTask) { calculateScheduler.scheduleAtFixedRate(currentCalculateTask::dispatch, 5, intervals, TimeUnit.SECONDS); } - private void startCombineHistory(ScheduledDispatchTask currentCombineHistoryTask) { + private void startCombineHistory(PausableDispatchTask currentCombineHistoryTask) { // combine history every day at 1:00 AM LocalDateTime now = LocalDateTime.now(); LocalDateTime nextRun = now.withHour(1).withMinute(0).withSecond(0); @@ -164,15 +234,23 @@ public class CalculateStatus implements DisposableBean { return intervals; } - synchronized void dispatchCalculate() { - if (calculateTask != null) { - calculateTask.dispatch(); + void dispatchCalculate() { + PausableDispatchTask currentTask; + synchronized (this) { + currentTask = calculateTask; + } + if (currentTask != null) { + currentTask.dispatch(); } } - synchronized void dispatchCombineHistory() { - if (combineHistoryTask != null) { - combineHistoryTask.dispatch(); + void dispatchCombineHistory() { + PausableDispatchTask currentTask; + synchronized (this) { + currentTask = combineHistoryTask; + } + if (currentTask != null) { + currentTask.dispatch(); } } @@ -360,90 +438,4 @@ public class CalculateStatus implements DisposableBean { }) .factory()); } - - private static final class ScheduledDispatchTask { - - private final ExecutorService executorService; - private final Runnable task; - private final Object lock = new Object(); - private boolean running; - private int pendingRuns; - private boolean cancelled; - - private ScheduledDispatchTask(ExecutorService executorService, Runnable task) { - this.executorService = executorService; - this.task = task; - } - - private void dispatch() { - if (executorService == null) { - synchronized (lock) { - if (cancelled) { - return; - } - } - task.run(); - return; - } - synchronized (lock) { - if (cancelled) { - return; - } - if (running) { - pendingRuns++; - return; - } - running = true; - } - submit(); - } - - private void submit() { - boolean submitted = false; - try { - executorService.execute(() -> { - try { - task.run(); - } finally { - onComplete(); - } - }); - submitted = true; - } finally { - if (!submitted) { - synchronized (lock) { - running = false; - pendingRuns = 0; - } - } - } - } - - private void onComplete() { - boolean shouldRunAgain; - synchronized (lock) { - if (cancelled) { - running = false; - pendingRuns = 0; - shouldRunAgain = false; - } else if (pendingRuns > 0) { - pendingRuns--; - shouldRunAgain = true; - } else { - running = false; - shouldRunAgain = false; - } - } - if (shouldRunAgain) { - submit(); - } - } - - private void cancel() { - synchronized (lock) { - cancelled = true; - pendingRuns = 0; - } - } - } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/PausableDispatchTask.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/PausableDispatchTask.java new file mode 100644 index 0000000000..48053a609a --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/PausableDispatchTask.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.component.status; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import org.apache.hertzbeat.manager.maintenance.MaintenanceDeadline; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceException; + +/** Single-flight dispatch with pause, drain, and one coalesced due run. */ +final class PausableDispatchTask { + + private final ExecutorService executorService; + private final Runnable task; + private final Object lock = new Object(); + private boolean running; + private boolean pendingRun; + private boolean paused; + private boolean missedWhilePaused; + private boolean cancelled; + + PausableDispatchTask(ExecutorService executorService, Runnable task) { + this.executorService = executorService; + this.task = task; + } + + void dispatch() { + synchronized (lock) { + if (cancelled) { + return; + } + if (paused) { + missedWhilePaused = true; + return; + } + if (running) { + pendingRun = true; + return; + } + running = true; + } + runOrSubmit(); + } + + void pauseAdmission() { + synchronized (lock) { + paused = true; + missedWhilePaused |= pendingRun; + pendingRun = false; + } + } + + void awaitDrained(MaintenanceDeadline deadline) { + synchronized (lock) { + while (running) { + long remainingNanos = deadline.remainingNanos(); + if (remainingNanos <= 0) { + throw MetadataMaintenanceException.quiesceTimeout(); + } + try { + TimeUnit.NANOSECONDS.timedWait(lock, remainingNanos); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw MetadataMaintenanceException.quiesceInterrupted(); + } + } + } + } + + void resumeAdmission() { + boolean shouldRun = false; + synchronized (lock) { + if (!paused) { + return; + } + paused = false; + if (missedWhilePaused) { + missedWhilePaused = false; + if (running) { + pendingRun = true; + } else if (!cancelled) { + running = true; + shouldRun = true; + } + } + lock.notifyAll(); + } + if (shouldRun) { + runOrSubmit(); + } + } + + void cancel() { + synchronized (lock) { + cancelled = true; + paused = true; + pendingRun = false; + missedWhilePaused = false; + lock.notifyAll(); + } + } + + private void runOrSubmit() { + if (executorService == null) { + try { + task.run(); + } finally { + onComplete(); + } + return; + } + boolean submitted = false; + try { + executorService.execute(() -> { + try { + task.run(); + } finally { + onComplete(); + } + }); + submitted = true; + } finally { + if (!submitted) { + synchronized (lock) { + running = false; + pendingRun = false; + lock.notifyAll(); + } + } + } + } + + private void onComplete() { + boolean shouldRunAgain; + synchronized (lock) { + if (cancelled || paused) { + running = false; + pendingRun = false; + shouldRunAgain = false; + } else if (pendingRun) { + pendingRun = false; + shouldRunAgain = true; + } else { + running = false; + shouldRunAgain = false; + } + lock.notifyAll(); + } + if (shouldRunAgain) { + runOrSubmit(); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MaintenanceDeadline.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MaintenanceDeadline.java new file mode 100644 index 0000000000..354a9c40f0 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MaintenanceDeadline.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import java.time.Duration; +import java.util.function.LongSupplier; + +/** Shared monotonic deadline for one process-local maintenance transition. */ +public final class MaintenanceDeadline { + + private final long timeoutNanos; + private final long startedNanos; + private final LongSupplier ticker; + + private MaintenanceDeadline(long timeoutNanos, long startedNanos, LongSupplier ticker) { + this.timeoutNanos = timeoutNanos; + this.startedNanos = startedNanos; + this.ticker = ticker; + } + + public static MaintenanceDeadline start(Duration timeout) { + return start(timeout, System::nanoTime); + } + + static MaintenanceDeadline start(Duration timeout, LongSupplier ticker) { + if (timeout == null || timeout.isNegative()) { + throw MetadataMaintenanceException.invalidRequest(); + } + try { + long timeoutNanos = timeout.toNanos(); + return new MaintenanceDeadline(timeoutNanos, ticker.getAsLong(), ticker); + } catch (ArithmeticException exception) { + throw MetadataMaintenanceException.invalidRequest(); + } + } + + public long remainingNanos() { + long elapsedNanos = ticker.getAsLong() - startedNanos; + if (elapsedNanos <= 0) { + return timeoutNanos; + } + if (elapsedNanos >= timeoutNanos) { + return 0; + } + return timeoutNanos - elapsedNanos; + } + + public Duration remaining() { + return Duration.ofNanos(remainingNanos()); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceCoordinator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceCoordinator.java new file mode 100644 index 0000000000..2f79338a83 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceCoordinator.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.locks.ReentrantLock; +import org.springframework.stereotype.Component; + +/** + * Coordinates process-local metadata producers without controlling their shared executors. + * + *

A future migration workflow must quiesce this coordinator before acquiring metadata write + * admission. On exit it must release write admission before resuming this lease. Keeping those + * capabilities separate prevents producer lifecycle from becoming a transaction or datasource + * switch.

+ */ +@Component +public final class MetadataMaintenanceCoordinator { + + private final ReentrantLock lock = new ReentrantLock(); + private final List participants; + private MetadataMaintenancePhase phase = MetadataMaintenancePhase.RUNNING; + private String operationId; + private long epoch; + private Object leaseToken; + + public MetadataMaintenanceCoordinator(List participants) { + this.participants = List.copyOf(participants); + validateParticipants(this.participants); + } + + /** Pause producers in registration order and drain work admitted before the pause. */ + public MetadataMaintenanceLease quiesce(String requestedOperationId, Duration timeout) { + MaintenanceDeadline deadline = MaintenanceDeadline.start(timeout); + requireOperationId(requestedOperationId); + Acquisition acquisition = beginAcquisition(requestedOperationId); + + List completed = new ArrayList<>(participants.size()); + try { + for (MetadataMaintenanceParticipant participant : participants) { + participant.quiesce(deadline.remaining()); + completed.add(participant); + } + } catch (MetadataMaintenanceException exception) { + rollback(acquisition, completed); + throw exception; + } catch (Error error) { + rollback(acquisition, completed); + throw error; + } catch (RuntimeException exception) { + rollback(acquisition, completed); + throw MetadataMaintenanceException.participantFailure(); + } + return completeAcquisition(acquisition); + } + + public MetadataMaintenanceSnapshot snapshot() { + lock.lock(); + try { + return new MetadataMaintenanceSnapshot(phase, operationId, epoch); + } finally { + lock.unlock(); + } + } + + void resume(String resumedOperationId, long resumedEpoch, Object resumedToken) { + lock.lock(); + try { + if (!ownsResumeLease(resumedOperationId, resumedEpoch, resumedToken)) { + throw MetadataMaintenanceException.staleLease(); + } + phase = MetadataMaintenancePhase.QUIESCING; + boolean failed = false; + for (int index = participants.size() - 1; index >= 0; index--) { + try { + participants.get(index).resume(); + } catch (RuntimeException exception) { + failed = true; + } + } + if (failed) { + throw MetadataMaintenanceException.resumeFailure(); + } + reopen(); + } finally { + lock.unlock(); + } + } + + private Acquisition beginAcquisition(String requestedOperationId) { + lock.lock(); + try { + if (phase != MetadataMaintenancePhase.RUNNING) { + throw MetadataMaintenanceException.operationConflict(); + } + phase = MetadataMaintenancePhase.QUIESCING; + operationId = requestedOperationId; + long requestedEpoch = ++epoch; + Object requestedToken = new Object(); + leaseToken = requestedToken; + return new Acquisition(requestedOperationId, requestedEpoch, requestedToken); + } finally { + lock.unlock(); + } + } + + private MetadataMaintenanceLease completeAcquisition(Acquisition acquisition) { + lock.lock(); + try { + if (!ownsAcquisition(acquisition)) { + throw MetadataMaintenanceException.operationConflict(); + } + phase = MetadataMaintenancePhase.QUIESCED; + return lease(acquisition); + } finally { + lock.unlock(); + } + } + + private void rollback(Acquisition acquisition, List completed) { + Collections.reverse(completed); + for (MetadataMaintenanceParticipant participant : completed) { + try { + participant.resume(); + } catch (RuntimeException exception) { + // Rollback is best effort and must not replace the primary safe failure category. + } + } + lock.lock(); + try { + if (ownsAcquisition(acquisition)) { + reopen(); + } + } finally { + lock.unlock(); + } + } + + private boolean ownsAcquisition(Acquisition acquisition) { + return phase == MetadataMaintenancePhase.QUIESCING + && epoch == acquisition.epoch() + && operationId.equals(acquisition.operationId()) + && leaseToken == acquisition.token(); + } + + private boolean ownsResumeLease(String resumedOperationId, long resumedEpoch, Object resumedToken) { + return (phase == MetadataMaintenancePhase.QUIESCED + || phase == MetadataMaintenancePhase.QUIESCING) + && epoch == resumedEpoch + && operationId.equals(resumedOperationId) + && leaseToken == resumedToken; + } + + private MetadataMaintenanceLease lease(Acquisition acquisition) { + return new MetadataMaintenanceLease( + this, acquisition.operationId(), acquisition.epoch(), acquisition.token()); + } + + private void reopen() { + phase = MetadataMaintenancePhase.RUNNING; + operationId = null; + leaseToken = null; + } + + private void requireOperationId(String requestedOperationId) { + if (requestedOperationId == null || requestedOperationId.isBlank()) { + throw MetadataMaintenanceException.invalidRequest(); + } + } + + private void validateParticipants(List registeredParticipants) { + Set participantIds = new HashSet<>(registeredParticipants.size()); + for (MetadataMaintenanceParticipant participant : registeredParticipants) { + String participantId = participant.participantId(); + if (participantId == null || participantId.isBlank() || !participantIds.add(participantId)) { + throw MetadataMaintenanceException.invalidRequest(); + } + } + } + + private record Acquisition(String operationId, long epoch, Object token) { + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceErrorCode.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceErrorCode.java new file mode 100644 index 0000000000..b4745b2ba4 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceErrorCode.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +/** Stable, secret-free control-plane maintenance failure classifications. */ +public enum MetadataMaintenanceErrorCode { + INVALID_REQUEST("invalid_request"), + OPERATION_CONFLICT("operation_conflict"), + QUIESCE_TIMEOUT("quiesce_timeout"), + QUIESCE_INTERRUPTED("quiesce_interrupted"), + PARTICIPANT_FAILURE("participant_failure"), + RESUME_FAILURE("resume_failure"), + STALE_LEASE("stale_lease"); + + private final String wireCode; + + MetadataMaintenanceErrorCode(String wireCode) { + this.wireCode = wireCode; + } + + public String wireCode() { + return wireCode; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceException.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceException.java new file mode 100644 index 0000000000..6bf08ff279 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceException.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +/** Safe maintenance failure that never exposes participant work or persistence details. */ +public final class MetadataMaintenanceException extends RuntimeException { + + private static final String INVALID_MESSAGE = "Metadata maintenance request is invalid"; + private static final String CONFLICT_MESSAGE = "Metadata maintenance operation is already active"; + private static final String TIMEOUT_MESSAGE = "Metadata producer drain timed out"; + private static final String INTERRUPTED_MESSAGE = "Metadata producer drain was interrupted"; + private static final String PARTICIPANT_MESSAGE = "Metadata producer could not be paused"; + private static final String RESUME_MESSAGE = "Metadata producer could not be resumed"; + private static final String STALE_MESSAGE = "Metadata maintenance lease is stale"; + + private final MetadataMaintenanceErrorCode code; + + private MetadataMaintenanceException(MetadataMaintenanceErrorCode code, String message) { + super(message); + this.code = code; + } + + public MetadataMaintenanceErrorCode code() { + return code; + } + + public String safeMessage() { + return getMessage(); + } + + static MetadataMaintenanceException invalidRequest() { + return new MetadataMaintenanceException(MetadataMaintenanceErrorCode.INVALID_REQUEST, INVALID_MESSAGE); + } + + static MetadataMaintenanceException operationConflict() { + return new MetadataMaintenanceException(MetadataMaintenanceErrorCode.OPERATION_CONFLICT, CONFLICT_MESSAGE); + } + + public static MetadataMaintenanceException quiesceTimeout() { + return new MetadataMaintenanceException(MetadataMaintenanceErrorCode.QUIESCE_TIMEOUT, TIMEOUT_MESSAGE); + } + + public static MetadataMaintenanceException quiesceInterrupted() { + return new MetadataMaintenanceException(MetadataMaintenanceErrorCode.QUIESCE_INTERRUPTED, INTERRUPTED_MESSAGE); + } + + static MetadataMaintenanceException participantFailure() { + return new MetadataMaintenanceException( + MetadataMaintenanceErrorCode.PARTICIPANT_FAILURE, PARTICIPANT_MESSAGE); + } + + static MetadataMaintenanceException resumeFailure() { + return new MetadataMaintenanceException(MetadataMaintenanceErrorCode.RESUME_FAILURE, RESUME_MESSAGE); + } + + static MetadataMaintenanceException staleLease() { + return new MetadataMaintenanceException(MetadataMaintenanceErrorCode.STALE_LEASE, STALE_MESSAGE); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceLease.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceLease.java new file mode 100644 index 0000000000..384594d41b --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceLease.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +/** Epoch-bound capability that resumes metadata producers exactly once. */ +public final class MetadataMaintenanceLease implements AutoCloseable { + + private final MetadataMaintenanceCoordinator coordinator; + private final String operationId; + private final long epoch; + private final Object token; + private boolean resumed; + + MetadataMaintenanceLease( + MetadataMaintenanceCoordinator coordinator, String operationId, long epoch, Object token) { + this.coordinator = coordinator; + this.operationId = operationId; + this.epoch = epoch; + this.token = token; + } + + public synchronized void resume() { + if (resumed) { + return; + } + coordinator.resume(operationId, epoch, token); + resumed = true; + } + + @Override + public void close() { + resume(); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceParticipant.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceParticipant.java new file mode 100644 index 0000000000..9a4ba90526 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceParticipant.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import java.time.Duration; + +/** One metadata producer that can stop admitting work and drain already admitted work. */ +public interface MetadataMaintenanceParticipant { + + String participantId(); + + /** Stop admitting work and drain work admitted before this call. */ + void quiesce(Duration timeout); + + /** Resume normal admission without creating another scheduler or consumer. */ + void resume(); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenancePhase.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenancePhase.java new file mode 100644 index 0000000000..e0997a4673 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenancePhase.java @@ -0,0 +1,15 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +/** Lifecycle of process-local metadata producer admission. */ +public enum MetadataMaintenancePhase { + RUNNING, + QUIESCING, + QUIESCED +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceSnapshot.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceSnapshot.java new file mode 100644 index 0000000000..ce770a8177 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceSnapshot.java @@ -0,0 +1,13 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +/** Read-only process-local maintenance state without the lease capability. */ +public record MetadataMaintenanceSnapshot( + MetadataMaintenancePhase phase, String operationId, long epoch) { +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryWorkerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryWorkerTest.java index 9ba4bf2e96..f2e84a4297 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryWorkerTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryWorkerTest.java @@ -17,8 +17,12 @@ package org.apache.hertzbeat.manager.component.sd; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -29,8 +33,16 @@ import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.read.ListAppender; +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.common.entity.message.CollectRep; import org.apache.hertzbeat.common.queue.CommonDataQueue; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceErrorCode; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceException; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenancePhase; import org.apache.hertzbeat.manager.dao.CollectorMonitorBindDao; import org.apache.hertzbeat.manager.dao.MonitorBindDao; import org.apache.hertzbeat.manager.dao.MonitorDao; @@ -115,7 +127,168 @@ class ServiceDiscoveryWorkerTest { } } + @Test + void quiesceWakesBlockedPollAndResumeKeepsOneConsumerLoop() throws Exception { + CommonDataQueue dataQueue = mock(CommonDataQueue.class); + CountDownLatch firstPoll = new CountDownLatch(1); + CollectRep.MetricsData resumedData = mock(CollectRep.MetricsData.class); + when(resumedData.getId()).thenReturn(42L); + when(dataQueue.pollServiceDiscoveryData()) + .thenAnswer(invocation -> { + firstPoll.countDown(); + new CountDownLatch(1).await(); + return null; + }) + .thenReturn(resumedData) + .thenAnswer(invocation -> { + new CountDownLatch(1).await(); + return null; + }); + MonitorDao monitorDao = mock(MonitorDao.class); + CountDownLatch consumedAfterResume = new CountDownLatch(1); + when(monitorDao.findById(42L)).thenAnswer(invocation -> { + consumedAfterResume.countDown(); + return Optional.empty(); + }); + WorkerHarness harness = captureHarness(dataQueue, monitorDao); + Thread consumer = Thread.ofPlatform().unstarted(harness.task()); + consumer.start(); + assertThat(firstPoll.await(1, TimeUnit.SECONDS)).isTrue(); + + harness.worker().quiesce(Duration.ofSeconds(1)); + + verify(dataQueue, times(1)).pollServiceDiscoveryData(); + assertThat(consumer.isAlive()).isTrue(); + verify(harness.workerPool(), times(1)).executeLongRunning(any(Runnable.class)); + harness.worker().resume(); + assertThat(consumedAfterResume.await(1, TimeUnit.SECONDS)).isTrue(); + consumer.interrupt(); + consumer.join(1_000); + assertThat(consumer.isAlive()).isFalse(); + verify(harness.workerPool(), times(1)).executeLongRunning(any(Runnable.class)); + } + + @Test + void quiesceWaitsForEnteredMessageAndStartsNoNewWork() throws Exception { + CommonDataQueue dataQueue = mock(CommonDataQueue.class); + CollectRep.MetricsData data = mock(CollectRep.MetricsData.class); + when(data.getId()).thenReturn(42L); + when(dataQueue.pollServiceDiscoveryData()).thenReturn(data).thenAnswer(invocation -> { + new CountDownLatch(1).await(); + return null; + }); + MonitorDao monitorDao = mock(MonitorDao.class); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + when(monitorDao.findById(42L)).thenAnswer(invocation -> { + entered.countDown(); + release.await(); + return Optional.empty(); + }); + WorkerHarness harness = captureHarness(dataQueue, monitorDao); + Thread consumer = Thread.ofPlatform().unstarted(harness.task()); + consumer.start(); + assertThat(entered.await(1, TimeUnit.SECONDS)).isTrue(); + CountDownLatch quiesced = new CountDownLatch(1); + AtomicReference failure = new AtomicReference<>(); + Thread controller = Thread.ofPlatform().unstarted(() -> { + try { + harness.worker().quiesce(Duration.ofSeconds(5)); + } catch (Throwable throwable) { + failure.set(throwable); + } finally { + quiesced.countDown(); + } + }); + controller.start(); + + awaitPhase(harness.worker(), MetadataMaintenancePhase.QUIESCING); + assertThat(quiesced.getCount()).isOne(); + release.countDown(); + assertThat(quiesced.await(1, TimeUnit.SECONDS)).isTrue(); + assertThat(failure.get()).isNull(); + verify(dataQueue, times(1)).pollServiceDiscoveryData(); + verify(data, times(1)).close(); + + harness.worker().resume(); + consumer.interrupt(); + consumer.join(1_000); + assertThat(consumer.isAlive()).isFalse(); + } + + @Test + void zeroTimeoutRestoresConsumptionAfterSafeFailure() throws Exception { + CommonDataQueue dataQueue = mock(CommonDataQueue.class); + CollectRep.MetricsData firstData = mock(CollectRep.MetricsData.class); + CollectRep.MetricsData secondData = mock(CollectRep.MetricsData.class); + when(firstData.getId()).thenReturn(42L); + when(secondData.getId()).thenReturn(43L); + when(dataQueue.pollServiceDiscoveryData()).thenReturn(firstData, secondData).thenAnswer(invocation -> { + new CountDownLatch(1).await(); + return null; + }); + MonitorDao monitorDao = mock(MonitorDao.class); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch consumedAfterTimeout = new CountDownLatch(1); + when(monitorDao.findById(42L)).thenAnswer(invocation -> { + entered.countDown(); + release.await(); + return Optional.empty(); + }); + when(monitorDao.findById(43L)).thenAnswer(invocation -> { + consumedAfterTimeout.countDown(); + return Optional.empty(); + }); + WorkerHarness harness = captureHarness(dataQueue, monitorDao); + Thread consumer = Thread.ofPlatform().unstarted(harness.task()); + consumer.start(); + assertThat(entered.await(1, TimeUnit.SECONDS)).isTrue(); + + assertThatThrownBy(() -> harness.worker().quiesce(Duration.ZERO)) + .isInstanceOfSatisfying(MetadataMaintenanceException.class, exception -> + assertThat(exception.code()).isEqualTo(MetadataMaintenanceErrorCode.QUIESCE_TIMEOUT)); + + release.countDown(); + assertThat(consumedAfterTimeout.await(1, TimeUnit.SECONDS)).isTrue(); + verify(dataQueue, atLeast(2)).pollServiceDiscoveryData(); + consumer.interrupt(); + consumer.join(1_000); + assertThat(consumer.isAlive()).isFalse(); + } + + @Test + void terminalDestroyStopsQuiescedLoopAndResumeCannotReviveIt() throws Exception { + CommonDataQueue dataQueue = mock(CommonDataQueue.class); + CountDownLatch pollEntered = new CountDownLatch(1); + when(dataQueue.pollServiceDiscoveryData()).thenAnswer(invocation -> { + pollEntered.countDown(); + new CountDownLatch(1).await(); + return null; + }); + WorkerHarness harness = captureHarness(dataQueue, mock(MonitorDao.class)); + Thread consumer = Thread.ofPlatform().unstarted(harness.task()); + consumer.start(); + assertThat(pollEntered.await(1, TimeUnit.SECONDS)).isTrue(); + harness.worker().quiesce(Duration.ofSeconds(1)); + assertThat(consumer.isAlive()).isTrue(); + + harness.worker().destroy(); + harness.worker().destroy(); + consumer.join(1_000); + assertThat(consumer.isAlive()).isFalse(); + + harness.worker().resume(); + harness.worker().resume(); + verify(dataQueue, times(1)).pollServiceDiscoveryData(); + verify(harness.workerPool(), times(1)).executeLongRunning(any(Runnable.class)); + } + private Runnable captureTask(CommonDataQueue dataQueue) { + return captureHarness(dataQueue, mock(MonitorDao.class)).task(); + } + + private WorkerHarness captureHarness(CommonDataQueue dataQueue, MonitorDao monitorDao) { ManagerWorkerPool workerPool = mock(ManagerWorkerPool.class); AtomicReference task = new AtomicReference<>(); doAnswer(invocation -> { @@ -125,13 +298,25 @@ class ServiceDiscoveryWorkerTest { ServiceDiscoveryWorker worker = new ServiceDiscoveryWorker( mock(MonitorService.class), mock(ParamDao.class), - mock(MonitorDao.class), + monitorDao, mock(MonitorBindDao.class), mock(CollectorMonitorBindDao.class), dataQueue, workerPool); worker.afterPropertiesSet(); - return task.get(); + return new WorkerHarness(worker, task.get(), workerPool); + } + + private void awaitPhase(ServiceDiscoveryWorker worker, MetadataMaintenancePhase expected) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(1); + while (worker.maintenancePhase() != expected && System.nanoTime() < deadline) { + Thread.onSpinWait(); + } + assertThat(worker.maintenancePhase()).isEqualTo(expected); + } + + private record WorkerHarness( + ServiceDiscoveryWorker worker, Runnable task, ManagerWorkerPool workerPool) { } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/status/CalculateStatusTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/status/CalculateStatusTest.java index 2e607cd84a..f0c13654f1 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/status/CalculateStatusTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/status/CalculateStatusTest.java @@ -17,12 +17,14 @@ package org.apache.hertzbeat.manager.component.status; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.verifyNoInteractions; +import java.time.Duration; import java.util.Collections; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -34,6 +36,7 @@ import org.apache.hertzbeat.manager.dao.MonitorDao; import org.apache.hertzbeat.manager.dao.StatusPageComponentDao; import org.apache.hertzbeat.manager.dao.StatusPageHistoryDao; import org.apache.hertzbeat.manager.dao.StatusPageOrgDao; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenancePhase; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -220,6 +223,141 @@ class CalculateStatusTest { assertEquals(1, invocations.get()); } + @Test + void quiesceAtomicallyDropsPendingRunAndWaitsForInFlightCompletion() throws Exception { + calculateStatus.start(); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch quiesced = new CountDownLatch(1); + AtomicInteger invocations = new AtomicInteger(); + org.mockito.Mockito.doAnswer(invocation -> { + invocations.incrementAndGet(); + entered.countDown(); + release.await(); + return Collections.emptyList(); + }).when(statusPageOrgDao).findAll(); + + calculateStatus.dispatchCalculate(); + assertThat(entered.await(1, TimeUnit.SECONDS)).isTrue(); + calculateStatus.dispatchCalculate(); + Thread controller = Thread.ofPlatform().unstarted(() -> { + calculateStatus.quiesce(Duration.ofSeconds(5)); + quiesced.countDown(); + }); + controller.start(); + + awaitPhase(MetadataMaintenancePhase.QUIESCING); + assertThat(quiesced.getCount()).isOne(); + release.countDown(); + assertThat(quiesced.await(1, TimeUnit.SECONDS)).isTrue(); + assertThat(invocations).hasValue(1); + + calculateStatus.dispatchCalculate(); + assertThat(invocations).hasValue(1); + } + + @Test + void resumeCoalescesPausedCalculateDispatchesIntoOneRun() throws Exception { + calculateStatus.start(); + calculateStatus.quiesce(Duration.ofSeconds(1)); + CountDownLatch invoked = new CountDownLatch(1); + AtomicInteger invocations = new AtomicInteger(); + org.mockito.Mockito.doAnswer(invocation -> { + invocations.incrementAndGet(); + invoked.countDown(); + return Collections.emptyList(); + }).when(statusPageOrgDao).findAll(); + + calculateStatus.dispatchCalculate(); + calculateStatus.dispatchCalculate(); + calculateStatus.resume(); + calculateStatus.resume(); + + assertThat(invoked.await(1, TimeUnit.SECONDS)).isTrue(); + assertThat(invocations).hasValue(1); + } + + @Test + void resumeCoalescesMissedDailyCombineIntoOneRun() throws Exception { + calculateStatus.start(); + calculateStatus.quiesce(Duration.ofSeconds(1)); + CountDownLatch invoked = new CountDownLatch(1); + AtomicInteger invocations = new AtomicInteger(); + org.mockito.Mockito.doAnswer(invocation -> { + invocations.incrementAndGet(); + invoked.countDown(); + return Collections.emptyList(); + }).when(statusPageHistoryDao).findStatusPageHistoriesByTimestampBetween(anyLong(), anyLong()); + + calculateStatus.dispatchCombineHistory(); + calculateStatus.dispatchCombineHistory(); + calculateStatus.resume(); + calculateStatus.resume(); + + assertThat(invoked.await(1, TimeUnit.SECONDS)).isTrue(); + assertThat(invocations).hasValue(1); + } + + @Test + void runningDailyCombinePreservesOnePendingDueRunAcrossMaintenance() throws Exception { + calculateStatus.start(); + CountDownLatch firstEntered = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondEntered = new CountDownLatch(1); + AtomicInteger invocations = new AtomicInteger(); + org.mockito.Mockito.doAnswer(invocation -> { + int invocationNumber = invocations.incrementAndGet(); + if (invocationNumber == 1) { + firstEntered.countDown(); + releaseFirst.await(); + } else { + secondEntered.countDown(); + } + return Collections.emptyList(); + }).when(statusPageHistoryDao).findStatusPageHistoriesByTimestampBetween(anyLong(), anyLong()); + + calculateStatus.dispatchCombineHistory(); + assertThat(firstEntered.await(1, TimeUnit.SECONDS)).isTrue(); + calculateStatus.dispatchCombineHistory(); + CountDownLatch quiesced = new CountDownLatch(1); + Thread controller = Thread.ofPlatform().unstarted(() -> { + calculateStatus.quiesce(Duration.ofSeconds(5)); + quiesced.countDown(); + }); + controller.start(); + awaitPhase(MetadataMaintenancePhase.QUIESCING); + assertThat(quiesced.getCount()).isOne(); + + releaseFirst.countDown(); + assertThat(quiesced.await(1, TimeUnit.SECONDS)).isTrue(); + assertThat(invocations).hasValue(1); + calculateStatus.resume(); + + assertThat(secondEntered.await(1, TimeUnit.SECONDS)).isTrue(); + assertThat(invocations).hasValue(2); + } + + @Test + void quiesceCanBeRepeatedWithoutDestroyingSchedulers() { + calculateStatus.start(); + + calculateStatus.quiesce(Duration.ofSeconds(1)); + calculateStatus.quiesce(Duration.ofSeconds(1)); + calculateStatus.resume(); + calculateStatus.resume(); + + assertThat(calculateStatus.isStarted()).isTrue(); + verifyNoInteractions(statusPageOrgDao, statusPageComponentDao, statusPageHistoryDao, monitorDao); + } + + private void awaitPhase(MetadataMaintenancePhase expected) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(1); + while (calculateStatus.maintenancePhase() != expected && System.nanoTime() < deadline) { + Thread.onSpinWait(); + } + assertThat(calculateStatus.maintenancePhase()).isEqualTo(expected); + } + private StatusProperties statusProperties() { StatusProperties statusProperties = new StatusProperties(); StatusProperties.CalculateProperties calculateProperties = new StatusProperties.CalculateProperties(); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/status/PausableDispatchTaskTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/status/PausableDispatchTaskTest.java new file mode 100644 index 0000000000..2717d491a0 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/status/PausableDispatchTaskTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.component.status; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hertzbeat.manager.maintenance.MaintenanceDeadline; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceErrorCode; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceException; +import org.junit.jupiter.api.Test; + +class PausableDispatchTaskTest { + + @Test + void pendingDueRunIsCoalescedAcrossPauseAndDrain() throws Exception { + CountDownLatch firstEntered = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondEntered = new CountDownLatch(1); + AtomicInteger invocations = new AtomicInteger(); + try (ExecutorService executor = Executors.newThreadPerTaskExecutor(Thread.ofVirtual().factory())) { + PausableDispatchTask task = new PausableDispatchTask(executor, () -> { + int invocation = invocations.incrementAndGet(); + if (invocation == 1) { + firstEntered.countDown(); + await(releaseFirst); + } else { + secondEntered.countDown(); + } + }); + + task.dispatch(); + assertThat(firstEntered.await(1, TimeUnit.SECONDS)).isTrue(); + task.dispatch(); + task.pauseAdmission(); + releaseFirst.countDown(); + task.awaitDrained(MaintenanceDeadline.start(Duration.ofSeconds(1))); + assertThat(invocations).hasValue(1); + + task.resumeAdmission(); + task.resumeAdmission(); + assertThat(secondEntered.await(1, TimeUnit.SECONDS)).isTrue(); + assertThat(invocations).hasValue(2); + } + } + + @Test + void synchronousModeUsesTheSameRunningAndDrainAccounting() throws Exception { + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + PausableDispatchTask task = new PausableDispatchTask(null, () -> { + entered.countDown(); + await(release); + }); + Thread runner = Thread.ofPlatform().unstarted(task::dispatch); + runner.start(); + assertThat(entered.await(1, TimeUnit.SECONDS)).isTrue(); + task.pauseAdmission(); + + assertThatThrownBy(() -> task.awaitDrained(MaintenanceDeadline.start(Duration.ZERO))) + .isInstanceOfSatisfying(MetadataMaintenanceException.class, exception -> + assertThat(exception.code()).isEqualTo(MetadataMaintenanceErrorCode.QUIESCE_TIMEOUT)); + + release.countDown(); + task.awaitDrained(MaintenanceDeadline.start(Duration.ofSeconds(1))); + runner.join(1_000); + assertThat(runner.isAlive()).isFalse(); + } + + private void await(CountDownLatch latch) { + try { + latch.await(); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(exception); + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/MaintenanceDeadlineTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/MaintenanceDeadlineTest.java new file mode 100644 index 0000000000..300f05bbb1 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/MaintenanceDeadlineTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Duration; +import java.util.concurrent.atomic.AtomicLong; +import org.junit.jupiter.api.Test; + +class MaintenanceDeadlineTest { + + @Test + void negativeTickerOriginDoesNotSaturateOrExpirePositiveTimeout() { + AtomicLong ticker = new AtomicLong(-1_000); + MaintenanceDeadline deadline = MaintenanceDeadline.start(Duration.ofNanos(50), ticker::get); + + assertThat(deadline.remainingNanos()).isEqualTo(50); + ticker.addAndGet(20); + assertThat(deadline.remainingNanos()).isEqualTo(30); + ticker.addAndGet(30); + assertThat(deadline.remainingNanos()).isZero(); + } + + @Test + void tickerWrapUsesMonotonicElapsedDifference() { + AtomicLong ticker = new AtomicLong(Long.MAX_VALUE - 5); + MaintenanceDeadline deadline = MaintenanceDeadline.start(Duration.ofNanos(20), ticker::get); + + ticker.set(Long.MIN_VALUE + 5); + + assertThat(deadline.remainingNanos()).isEqualTo(9); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceCoordinatorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceCoordinatorTest.java new file mode 100644 index 0000000000..525eb2cb2c --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceCoordinatorTest.java @@ -0,0 +1,378 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; + +class MetadataMaintenanceCoordinatorTest { + + @Test + void rejectsDuplicateParticipantIds() { + List events = new ArrayList<>(); + + assertThatThrownBy(() -> new MetadataMaintenanceCoordinator(List.of( + participant("duplicate", events), participant("duplicate", events)))) + .isInstanceOfSatisfying(MetadataMaintenanceException.class, exception -> + assertThat(exception.code()).isEqualTo(MetadataMaintenanceErrorCode.INVALID_REQUEST)); + } + + @Test + void quiescesInOrderAndResumesInReverseOrder() { + List events = new ArrayList<>(); + MetadataMaintenanceCoordinator coordinator = new MetadataMaintenanceCoordinator(List.of( + participant("discovery", events), participant("status", events))); + + MetadataMaintenanceLease lease = coordinator.quiesce("operation-a", Duration.ofSeconds(1)); + + assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataMaintenancePhase.QUIESCED); + assertThat(events).containsExactly("pause-discovery", "pause-status"); + + lease.resume(); + + assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataMaintenancePhase.RUNNING); + assertThat(events).containsExactly( + "pause-discovery", "pause-status", "resume-status", "resume-discovery"); + } + + @Test + void duplicateOperationCannotResumeTheOwnerLease() { + List events = new ArrayList<>(); + MetadataMaintenanceCoordinator coordinator = new MetadataMaintenanceCoordinator( + List.of(participant("discovery", events))); + + MetadataMaintenanceLease owner = coordinator.quiesce("operation-a", Duration.ofSeconds(1)); + + assertThat(events).containsExactly("pause-discovery"); + assertThatThrownBy(() -> coordinator.quiesce("operation-a", Duration.ofSeconds(1))) + .isInstanceOfSatisfying(MetadataMaintenanceException.class, exception -> + assertThat(exception.code()).isEqualTo(MetadataMaintenanceErrorCode.OPERATION_CONFLICT)); + assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataMaintenancePhase.QUIESCED); + + owner.resume(); + owner.resume(); + assertThat(events).containsExactly("pause-discovery", "resume-discovery"); + assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataMaintenancePhase.RUNNING); + } + + @Test + void duplicateOperationWhileQuiescingCannotLaterObtainAnAliasLease() throws Exception { + CountDownLatch pauseEntered = new CountDownLatch(1); + CountDownLatch releasePause = new CountDownLatch(1); + MetadataMaintenanceParticipant participant = new MetadataMaintenanceParticipant() { + @Override + public String participantId() { + return "blocked-pause"; + } + + @Override + public void quiesce(Duration timeout) { + pauseEntered.countDown(); + await(releasePause); + } + + @Override + public void resume() { + } + }; + MetadataMaintenanceCoordinator coordinator = new MetadataMaintenanceCoordinator(List.of(participant)); + AtomicReference ownerLease = new AtomicReference<>(); + Thread ownerThread = Thread.ofPlatform().unstarted(() -> + ownerLease.set(coordinator.quiesce("operation-a", Duration.ofSeconds(30)))); + ownerThread.start(); + assertThat(pauseEntered.await(1, TimeUnit.SECONDS)).isTrue(); + + AtomicReference duplicateLease = new AtomicReference<>(); + AtomicReference duplicateFailure = new AtomicReference<>(); + Thread duplicateThread = Thread.ofPlatform().unstarted(() -> { + try { + duplicateLease.set(coordinator.quiesce("operation-a", Duration.ofSeconds(30))); + } catch (MetadataMaintenanceException exception) { + duplicateFailure.set(exception); + } + }); + duplicateThread.start(); + releasePause.countDown(); + ownerThread.join(1_000); + duplicateThread.join(1_000); + + assertThat(ownerThread.isAlive()).isFalse(); + assertThat(duplicateThread.isAlive()).isFalse(); + assertThat(duplicateLease.get()).isNull(); + assertThat(duplicateFailure.get().code()).isEqualTo(MetadataMaintenanceErrorCode.OPERATION_CONFLICT); + assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataMaintenancePhase.QUIESCED); + + ownerLease.get().resume(); + assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataMaintenancePhase.RUNNING); + } + + @Test + void conflictingOperationFailsWithoutDisclosingItsIdentifier() { + MetadataMaintenanceCoordinator coordinator = new MetadataMaintenanceCoordinator(List.of()); + coordinator.quiesce("private-operation", Duration.ZERO); + + assertThatThrownBy(() -> coordinator.quiesce("other-private-operation", Duration.ZERO)) + .isInstanceOfSatisfying(MetadataMaintenanceException.class, exception -> { + assertThat(exception.code()).isEqualTo(MetadataMaintenanceErrorCode.OPERATION_CONFLICT); + assertThat(exception.safeMessage()).doesNotContain("private"); + assertThat(exception.getCause()).isNull(); + }); + } + + @Test + void participantFailureResumesCompletedParticipantsInReverseOrder() { + List events = new ArrayList<>(); + MetadataMaintenanceParticipant first = participant("first", events); + MetadataMaintenanceParticipant failing = new MetadataMaintenanceParticipant() { + @Override + public String participantId() { + return "failing"; + } + + @Override + public void quiesce(Duration timeout) { + events.add("pause-failing"); + throw new IllegalStateException("private-task-body"); + } + + @Override + public void resume() { + events.add("resume-failing"); + } + }; + MetadataMaintenanceCoordinator coordinator = new MetadataMaintenanceCoordinator(List.of(first, failing)); + + assertThatThrownBy(() -> coordinator.quiesce("operation-a", Duration.ofSeconds(1))) + .isInstanceOfSatisfying(MetadataMaintenanceException.class, exception -> { + assertThat(exception.code()).isEqualTo(MetadataMaintenanceErrorCode.PARTICIPANT_FAILURE); + assertThat(exception.safeMessage()).doesNotContain("private-task-body"); + assertThat(exception.getCause()).isNull(); + }); + + assertThat(events).containsExactly("pause-first", "pause-failing", "resume-first"); + assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataMaintenancePhase.RUNNING); + } + + @Test + void timeoutRollsBackAndUsesStableSafeFailure() { + List events = new ArrayList<>(); + MetadataMaintenanceParticipant timeout = new MetadataMaintenanceParticipant() { + @Override + public String participantId() { + return "timeout"; + } + + @Override + public void quiesce(Duration ignored) { + throw MetadataMaintenanceException.quiesceTimeout(); + } + + @Override + public void resume() { + events.add("unexpected-resume"); + } + }; + MetadataMaintenanceCoordinator coordinator = new MetadataMaintenanceCoordinator(List.of(timeout)); + + assertThatThrownBy(() -> coordinator.quiesce("operation-a", Duration.ZERO)) + .isInstanceOfSatisfying(MetadataMaintenanceException.class, exception -> { + assertThat(exception.code()).isEqualTo(MetadataMaintenanceErrorCode.QUIESCE_TIMEOUT); + assertThat(exception.getCause()).isNull(); + }); + assertThat(events).isEmpty(); + assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataMaintenancePhase.RUNNING); + } + + @Test + void interruptedQuiesceRestoresInterruptAndRollsBack() throws Exception { + CountDownLatch entered = new CountDownLatch(1); + MetadataMaintenanceParticipant interruptible = new MetadataMaintenanceParticipant() { + @Override + public String participantId() { + return "interruptible"; + } + + @Override + public void quiesce(Duration ignored) { + entered.countDown(); + try { + new CountDownLatch(1).await(); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw MetadataMaintenanceException.quiesceInterrupted(); + } + } + + @Override + public void resume() { + } + }; + MetadataMaintenanceCoordinator coordinator = new MetadataMaintenanceCoordinator(List.of(interruptible)); + AtomicReference failure = new AtomicReference<>(); + AtomicReference interrupted = new AtomicReference<>(false); + Thread thread = Thread.ofPlatform().unstarted(() -> { + try { + coordinator.quiesce("operation-a", Duration.ofSeconds(30)); + } catch (MetadataMaintenanceException exception) { + failure.set(exception); + interrupted.set(Thread.currentThread().isInterrupted()); + } + }); + + thread.start(); + assertThat(entered.await(1, TimeUnit.SECONDS)).isTrue(); + thread.interrupt(); + thread.join(1_000); + + assertThat(thread.isAlive()).isFalse(); + assertThat(failure.get().code()).isEqualTo(MetadataMaintenanceErrorCode.QUIESCE_INTERRUPTED); + assertThat(interrupted.get()).isTrue(); + assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataMaintenancePhase.RUNNING); + } + + @Test + void resumeInvokesEachParticipantExactlyOnce() { + AtomicInteger resumes = new AtomicInteger(); + MetadataMaintenanceParticipant participant = new MetadataMaintenanceParticipant() { + @Override + public String participantId() { + return "counted"; + } + + @Override + public void quiesce(Duration timeout) { + } + + @Override + public void resume() { + resumes.incrementAndGet(); + } + }; + MetadataMaintenanceCoordinator coordinator = new MetadataMaintenanceCoordinator(List.of(participant)); + + MetadataMaintenanceLease lease = coordinator.quiesce("operation-a", Duration.ZERO); + lease.resume(); + lease.resume(); + + assertThat(resumes).hasValue(1); + } + + @Test + void resumeDoesNotExposePartiallyRunningParticipantsOrIssueAnotherLease() throws Exception { + CountDownLatch resumeEntered = new CountDownLatch(1); + CountDownLatch releaseResume = new CountDownLatch(1); + MetadataMaintenanceParticipant participant = new MetadataMaintenanceParticipant() { + @Override + public String participantId() { + return "blocked-resume"; + } + + @Override + public void quiesce(Duration timeout) { + } + + @Override + public void resume() { + resumeEntered.countDown(); + try { + releaseResume.await(); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw MetadataMaintenanceException.quiesceInterrupted(); + } + } + }; + MetadataMaintenanceCoordinator coordinator = new MetadataMaintenanceCoordinator(List.of(participant)); + MetadataMaintenanceLease lease = coordinator.quiesce("operation-a", Duration.ofSeconds(1)); + Thread resumeThread = Thread.ofPlatform().unstarted(lease::resume); + resumeThread.start(); + assertThat(resumeEntered.await(1, TimeUnit.SECONDS)).isTrue(); + + CountDownLatch acquisitionAttempted = new CountDownLatch(1); + CountDownLatch acquisitionReturned = new CountDownLatch(1); + AtomicReference laterLease = new AtomicReference<>(); + Thread acquisitionThread = Thread.ofPlatform().unstarted(() -> { + acquisitionAttempted.countDown(); + laterLease.set(coordinator.quiesce("operation-a", Duration.ofSeconds(1))); + acquisitionReturned.countDown(); + }); + acquisitionThread.start(); + assertThat(acquisitionAttempted.await(1, TimeUnit.SECONDS)).isTrue(); + assertThat(acquisitionReturned.getCount()).isOne(); + + releaseResume.countDown(); + resumeThread.join(1_000); + assertThat(acquisitionReturned.await(1, TimeUnit.SECONDS)).isTrue(); + laterLease.get().resume(); + } + + @Test + void virtualMachineErrorsAreNotMappedOrHidden() { + List events = new ArrayList<>(); + AssertionError fatal = new AssertionError("fatal"); + MetadataMaintenanceParticipant participant = new MetadataMaintenanceParticipant() { + @Override + public String participantId() { + return "fatal"; + } + + @Override + public void quiesce(Duration timeout) { + throw fatal; + } + + @Override + public void resume() { + } + }; + MetadataMaintenanceCoordinator coordinator = new MetadataMaintenanceCoordinator( + List.of(participant("first", events), participant)); + + assertThatThrownBy(() -> coordinator.quiesce("operation-a", Duration.ofSeconds(1))) + .isSameAs(fatal); + assertThat(events).containsExactly("pause-first", "resume-first"); + assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataMaintenancePhase.RUNNING); + } + + private MetadataMaintenanceParticipant participant(String name, List events) { + return new MetadataMaintenanceParticipant() { + @Override + public String participantId() { + return name; + } + + @Override + public void quiesce(Duration timeout) { + events.add("pause-" + name); + } + + @Override + public void resume() { + events.add("resume-" + name); + } + }; + } + + private void await(CountDownLatch latch) { + try { + latch.await(); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw MetadataMaintenanceException.quiesceInterrupted(); + } + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/MetadataWriteAdmissionStartupContextTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/MetadataWriteAdmissionStartupContextTest.java index 1791a87509..7c8f82d658 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/MetadataWriteAdmissionStartupContextTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/MetadataWriteAdmissionStartupContextTest.java @@ -11,9 +11,14 @@ import static org.assertj.core.api.Assertions.assertThat; import java.lang.reflect.Method; import java.util.Arrays; +import java.util.List; import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionAdvisor; import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionCoordinator; +import org.apache.hertzbeat.manager.component.sd.ServiceDiscoveryWorker; +import org.apache.hertzbeat.manager.component.status.CalculateStatus; import org.apache.hertzbeat.manager.dao.MonitorDao; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceCoordinator; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceParticipant; import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; import org.apache.hertzbeat.warehouse.store.DataStorageDispatch; import org.apache.hertzbeat.warehouse.store.metadata.JdbcMonitorStatusMetadataWriter; @@ -44,6 +49,15 @@ class MetadataWriteAdmissionStartupContextTest { @Test void startupHasOneAdmissionBoundaryAndOneTransactionSource() throws Exception { + assertThat(context.getBeansOfType(MetadataMaintenanceCoordinator.class)).hasSize(1); + List participants = context.getBeanProvider( + MetadataMaintenanceParticipant.class).orderedStream().toList(); + assertThat(participants) + .containsExactly( + context.getBean(ServiceDiscoveryWorker.class), + context.getBean(CalculateStatus.class)); + assertThat(participants).noneMatch(DataStorageDispatch.class::isInstance); + assertThat(context.getBeansOfType(MetadataWriteAdmissionCoordinator.class)).hasSize(1); assertThat(context.getBeansOfType(MetadataWriteAdmissionAdvisor.class)).hasSize(1); assertThat(context.getBeansOfType(TransactionAttributeSource.class)).hasSize(1); From dd3039611c221ed5217674bbfff66594dde1e271 Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 02:09:34 +0800 Subject: [PATCH 37/71] Quiesce collector and alert metadata writers --- .../periodic/PeriodicAlertRuleScheduler.java | 154 ++++++++++--- .../MetricsRealTimeAlertCalculator.java | 4 +- .../alert/notice/AlertNoticeDispatch.java | 35 ++- .../alert/reduce/AlarmCommonReduce.java | 111 ++++++++- .../alert/reduce/AlarmGroupReduce.java | 153 ++++++++++--- .../alert/reduce/AlarmInhibitReduce.java | 13 +- .../alert/reduce/AlarmSilenceReduce.java | 8 +- .../PeriodicAlertRuleSchedulerTest.java | 78 ++++++- ...tricsRealTimeAlertCalculatorMatchTest.java | 63 ++++++ .../alert/notice/AlertNoticeDispatchTest.java | 16 ++ .../alert/reduce/AlarmCommonReduceTest.java | 211 ++++++++++++++++++ .../alert/reduce/AlarmGroupReduceTest.java | 101 +++++++++ .../alert/reduce/AlarmInhibitReduceTest.java | 10 + .../common/concurrent/WorkAdmissionGate.java | 153 +++++++++++++ .../concurrent/WorkAdmissionGateTest.java | 122 ++++++++++ .../AlertMetadataMaintenanceParticipant.java | 117 ++++++++++ ...lectorLifecycleMaintenanceParticipant.java | 211 ++++++++++++++++++ .../MetadataMaintenanceCoordinator.java | 44 +++- .../maintenance/MetadataMaintenancePhase.java | 3 +- .../manager/scheduler/netty/ManageServer.java | 43 +++- .../process/CollectorOfflineProcessor.java | 2 +- .../process/CollectorOnlineProcessor.java | 2 +- .../netty/process/HeartbeatProcessor.java | 2 +- ...ertMetadataMaintenanceParticipantTest.java | 166 ++++++++++++++ ...orLifecycleMaintenanceParticipantTest.java | 203 +++++++++++++++++ .../MetadataMaintenanceCoordinatorTest.java | 34 ++- ...adataWriteAdmissionStartupContextTest.java | 6 +- .../StartupRuntimeBoundaryContextTest.java | 1 + 28 files changed, 1953 insertions(+), 113 deletions(-) create mode 100644 hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/WorkAdmissionGate.java create mode 100644 hertzbeat-common-core/src/test/java/org/apache/hertzbeat/common/concurrent/WorkAdmissionGateTest.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/AlertMetadataMaintenanceParticipant.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/CollectorLifecycleMaintenanceParticipant.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/AlertMetadataMaintenanceParticipantTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/CollectorLifecycleMaintenanceParticipantTest.java diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java index e3c4850ec0..e22d8ac69f 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java @@ -37,6 +37,7 @@ import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.alert.dao.AlertDefineDao; import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.apache.hertzbeat.common.concurrent.WorkAdmissionGate; import org.apache.hertzbeat.common.entity.alerter.AlertDefine; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -58,6 +59,8 @@ public class PeriodicAlertRuleScheduler { private final VirtualThreadProperties virtualThreadProperties; private boolean virtualThreadsEnabled; private final Map scheduledTasks; + private final WorkAdmissionGate maintenanceGate = new WorkAdmissionGate(); + private boolean maintenancePaused; @Autowired public PeriodicAlertRuleScheduler(MetricsPeriodicAlertCalculator metricsCalculator, @@ -152,7 +155,7 @@ public class PeriodicAlertRuleScheduler { ScheduledTaskState state = new ScheduledTaskState( rule, currentPeriodicExecutor, currentPeriodicPermits); ScheduledFuture future = currentScheduledExecutor.scheduleAtFixedRate( - virtualThreadsEnabled ? state::trigger : () -> executeRule(rule), + state::trigger, 0, rule.getPeriod(), TimeUnit.SECONDS); state.setScheduledFuture(future); scheduledTasks.put(rule.getId(), state); @@ -169,6 +172,48 @@ public class PeriodicAlertRuleScheduler { } } + public synchronized void pauseAdmission() { + maintenancePaused = true; + maintenanceGate.pauseAdmission(); + } + + public void awaitDrained(long timeoutNanos) throws InterruptedException, java.util.concurrent.TimeoutException { + maintenanceGate.awaitDrained(timeoutNanos); + } + + public void resumeAdmission() { + List states; + synchronized (this) { + if (!maintenancePaused) { + return; + } + maintenanceGate.resumeAdmission(); + maintenancePaused = false; + states = new ArrayList<>(scheduledTasks.values()); + } + states.forEach(ScheduledTaskState::resumeMissed); + } + + private synchronized WorkAdmissionGate.Permit acquireTriggerPermit(ScheduledTaskState state) { + if (maintenancePaused) { + state.markMissed(); + return null; + } + return maintenanceGate.tryAcquire(); + } + + void beforeRuleTrigger(AlertDefine rule) { + } + + private void executeRuleWithPermit(AlertDefine rule, WorkAdmissionGate.Permit permit) { + if (permit == null) { + return; + } + try (permit) { + executeRule(rule); + } + } + private boolean isPeriodicRule(String type) { return METRIC_ALERT_THRESHOLD_TYPE_PERIODIC.equals(type) || LOG_ALERT_THRESHOLD_TYPE_PERIODIC.equals(type) @@ -184,7 +229,9 @@ public class PeriodicAlertRuleScheduler { private Future runningFuture; private boolean running; private boolean pending; + private WorkAdmissionGate.Permit pendingPermit; private boolean cancelled; + private boolean missedWhilePaused; private ScheduledTaskState(AlertDefine rule, ExecutorService taskExecutor, Semaphore taskPermits) { this.rule = rule; @@ -196,21 +243,39 @@ public class PeriodicAlertRuleScheduler { this.scheduledFuture = scheduledFuture; } - private synchronized void trigger() { - if (cancelled) { + private void trigger() { + beforeRuleTrigger(rule); + WorkAdmissionGate.Permit permit = acquireTriggerPermit(this); + if (permit == null) { return; } - if (running) { - pending = true; - return; + synchronized (this) { + if (cancelled) { + permit.close(); + return; + } + if (running) { + if (!pending) { + pending = true; + pendingPermit = permit; + } else { + permit.close(); + } + return; + } + running = true; } - running = true; - submitLocked(); + submit(permit); } private synchronized void cancel() { cancelled = true; pending = false; + if (pendingPermit != null) { + pendingPermit.close(); + pendingPermit = null; + } + missedWhilePaused = false; ScheduledFuture periodicFuture = scheduledFuture; Future currentFuture = runningFuture; if (periodicFuture != null) { @@ -221,33 +286,46 @@ public class PeriodicAlertRuleScheduler { } } - private void submitLocked() { + private void submit(WorkAdmissionGate.Permit permit) { + if (taskExecutor == null) { + runTask(permit); + return; + } try { - runningFuture = taskExecutor.submit(() -> { - boolean permitAcquired = false; - try { - taskPermits.acquire(); - permitAcquired = true; - if (!Thread.currentThread().isInterrupted()) { - executeRule(rule); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } catch (Exception e) { - log.error("Periodic alert rule {} execution error: {}", rule.getName(), e.getMessage(), e); - } finally { - if (permitAcquired) { - taskPermits.release(); - } - onComplete(); - } - }); + runningFuture = taskExecutor.submit(() -> runTask(permit)); } catch (RuntimeException e) { running = false; + permit.close(); throw e; } } + private void runTask(WorkAdmissionGate.Permit permit) { + boolean concurrencyPermitAcquired = false; + try { + if (taskPermits != null) { + taskPermits.acquire(); + concurrencyPermitAcquired = true; + } + if (!Thread.currentThread().isInterrupted()) { + executeRuleWithPermit(rule, permit); + permit = null; + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Exception e) { + log.error("Periodic alert rule {} execution error: {}", rule.getName(), e.getMessage(), e); + } finally { + if (permit != null) { + permit.close(); + } + if (concurrencyPermitAcquired) { + taskPermits.release(); + } + onComplete(); + } + } + private synchronized void onComplete() { runningFuture = null; if (cancelled) { @@ -260,7 +338,25 @@ public class PeriodicAlertRuleScheduler { return; } pending = false; - submitLocked(); + WorkAdmissionGate.Permit nextPermit = pendingPermit; + pendingPermit = null; + submit(nextPermit); + } + + private synchronized void markMissed() { + if (!cancelled) { + missedWhilePaused = true; + } + } + + private void resumeMissed() { + synchronized (this) { + if (!missedWhilePaused || cancelled) { + return; + } + missedWhilePaused = false; + } + trigger(); } } } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/MetricsRealTimeAlertCalculator.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/MetricsRealTimeAlertCalculator.java index f675de1bae..0cad379152 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/MetricsRealTimeAlertCalculator.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/MetricsRealTimeAlertCalculator.java @@ -137,8 +137,10 @@ public class MetricsRealTimeAlertCalculator { continue; } backoff.reset(); - calculate(metricsData); + // The telemetry handoff precedes alert reduction so maintenance backpressure cannot + // occupy every calculator before later samples reach storage. dataQueue.sendMetricsDataToStorage(metricsData); + calculate(metricsData); } catch (InterruptedException ignored) { Thread.currentThread().interrupt(); } catch (CommonDataQueueUnknownException ue) { diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatch.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatch.java index c7c4de59fa..28d2f83909 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatch.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatch.java @@ -106,18 +106,29 @@ public class AlertNoticeDispatch { return Optional.ofNullable(noticeConfigService.getReceiverFilterRule(alert)); } - public void dispatchAlarm(GroupAlert groupAlert) { - if (groupAlert != null) { - // Determining alarm type storage - GroupAlert storedGroupAlert = alertStoreHandler.store(groupAlert); - // Notice distribution - sendNotify(storedGroupAlert); - // Execute the plugin if enable (Compatible with old version plugins, will be removed in later versions) - pluginRunner.pluginExecute(Plugin.class, plugin -> plugin.alert(storedGroupAlert)); - // Execute the plugin if enable with params - pluginRunner.pluginExecute(PostAlertPlugin.class, (afterAlertPlugin, pluginContext) -> afterAlertPlugin.execute(storedGroupAlert, pluginContext)); - // Send alert to the sse client - emitterManager.broadcast(JsonUtil.toJson(storedGroupAlert)); + public boolean dispatchAlarm(GroupAlert groupAlert) { + if (groupAlert == null) { + return false; + } + GroupAlert storedGroupAlert = alertStoreHandler.store(groupAlert); + dispatchAfterStore(storedGroupAlert); + return true; + } + + private void dispatchAfterStore(GroupAlert storedGroupAlert) { + runAfterStore(() -> sendNotify(storedGroupAlert), "notice"); + runAfterStore(() -> pluginRunner.pluginExecute( + Plugin.class, plugin -> plugin.alert(storedGroupAlert)), "legacy-plugin"); + runAfterStore(() -> pluginRunner.pluginExecute(PostAlertPlugin.class, + (plugin, context) -> plugin.execute(storedGroupAlert, context)), "post-plugin"); + runAfterStore(() -> emitterManager.broadcast(JsonUtil.toJson(storedGroupAlert)), "sse"); + } + + private void runAfterStore(Runnable action, String stage) { + try { + action.run(); + } catch (RuntimeException exception) { + log.warn("Post-store alert dispatch failed at stage: {}", stage); } } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduce.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduce.java index 64807ed8b3..716de2c410 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduce.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduce.java @@ -18,12 +18,17 @@ package org.apache.hertzbeat.alert.reduce; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.common.concurrent.ManagedExecutor; import org.apache.hertzbeat.common.concurrent.ManagedExecutors; +import org.apache.hertzbeat.common.concurrent.WorkAdmissionGate; import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; import org.springframework.beans.factory.DisposableBean; @@ -41,6 +46,14 @@ public class AlarmCommonReduce implements DisposableBean { private final ManagedExecutor workerExecutor; + private final WorkAdmissionGate maintenanceGate = new WorkAdmissionGate(); + + private final ReentrantLock maintenanceLock = new ReentrantLock(true); + + private final Deque deferredTasks = new ArrayDeque<>(); + + private boolean stopped; + public AlarmCommonReduce(AlarmGroupReduce alarmGroupReduce) { this(alarmGroupReduce, VirtualThreadProperties.defaults()); } @@ -53,6 +66,11 @@ public class AlarmCommonReduce implements DisposableBean { this.workerExecutor = initWorkExecutor(properties); } + AlarmCommonReduce(AlarmGroupReduce alarmGroupReduce, ManagedExecutor workerExecutor) { + this.alarmGroupReduce = alarmGroupReduce; + this.workerExecutor = workerExecutor; + } + private ManagedExecutor initWorkExecutor(VirtualThreadProperties properties) { Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { log.error("alerter-reduce-worker has uncaughtException."); @@ -78,11 +96,11 @@ public class AlarmCommonReduce implements DisposableBean { public void reduceAndSendAlarm(SingleAlert alert) { - workerExecutor.execute(reduceAlarmTask(alert)); + submitOrDefer(reduceAlarmTask(alert)); } public void reduceAndSendAlarmGroup(Map groupLabels, List alerts) { - workerExecutor.execute(() -> { + submitOrDefer(() -> { try { // Generate alert fingerprint for (SingleAlert alert : alerts) { @@ -97,6 +115,87 @@ public class AlarmCommonReduce implements DisposableBean { }); } + public void pauseAdmission() { + maintenanceLock.lock(); + try { + maintenanceGate.pauseAdmission(); + } finally { + maintenanceLock.unlock(); + } + } + + public void awaitDrained(long timeoutNanos) throws InterruptedException, TimeoutException { + maintenanceGate.awaitDrained(timeoutNanos); + } + + public void resumeAdmission() { + maintenanceLock.lock(); + try { + if (stopped) { + return; + } + while (!deferredTasks.isEmpty()) { + Runnable deferred = deferredTasks.peekFirst(); + WorkAdmissionGate.Permit permit = maintenanceGate.reserveReplay(); + if (permit == null) { + return; + } + submitAdmitted(deferred, permit); + deferredTasks.removeFirst(); + } + maintenanceGate.resumeAdmission(); + } finally { + maintenanceLock.unlock(); + } + } + + private void submitOrDefer(Runnable task) { + maintenanceLock.lock(); + try { + if (stopped) { + return; + } + WorkAdmissionGate.Permit permit = maintenanceGate.tryAcquire(); + if (permit != null) { + beforeAdmittedSubmission(); + submitAdmitted(task, permit); + return; + } + deferredTasks.addLast(task); + } finally { + maintenanceLock.unlock(); + } + } + + void beforeAdmittedSubmission() { + } + + boolean hasQueuedMaintenanceThread(Thread thread) { + return maintenanceLock.hasQueuedThread(thread); + } + + int deferredTaskCount() { + maintenanceLock.lock(); + try { + return deferredTasks.size(); + } finally { + maintenanceLock.unlock(); + } + } + + private void submitAdmitted(Runnable task, WorkAdmissionGate.Permit permit) { + try { + workerExecutor.execute(() -> { + try (permit) { + task.run(); + } + }); + } catch (RuntimeException exception) { + permit.close(); + throw exception; + } + } + Runnable reduceAlarmTask(SingleAlert alert) { return () -> { try { @@ -127,6 +226,14 @@ public class AlarmCommonReduce implements DisposableBean { @Override public void destroy() { + maintenanceLock.lock(); + try { + maintenanceGate.stop(); + deferredTasks.clear(); + stopped = true; + } finally { + maintenanceLock.unlock(); + } workerExecutor.close(); } } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java index 48040dc402..6268b790ba 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java @@ -31,6 +31,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -143,6 +144,36 @@ public class AlarmGroupReduce implements DisposableBean { void beforeCheckAndSendGroupsRun() { } + public void pauseAdmission() { + ScheduledDispatchTask currentTask; + synchronized (this) { + currentTask = checkTask; + } + if (currentTask != null) { + currentTask.pauseAdmission(); + } + } + + public void awaitDrained(long timeoutNanos) throws InterruptedException, TimeoutException { + ScheduledDispatchTask currentTask; + synchronized (this) { + currentTask = checkTask; + } + if (currentTask != null) { + currentTask.awaitDrained(timeoutNanos); + } + } + + public void resumeAdmission() { + ScheduledDispatchTask currentTask; + synchronized (this) { + currentTask = checkTask; + } + if (currentTask != null) { + currentTask.resumeAdmission(); + } + } + @Override public synchronized void destroy() { if (checkTask != null) { @@ -191,8 +222,6 @@ public class AlarmGroupReduce implements DisposableBean { groupCacheMap.forEach((groupKey, cache) -> { if (shouldSendGroup(cache, now)) { sendGroupAlert(cache); - cache.setLastSendTime(now); - cache.getAlertFingerprints().clear(); } }); } catch (Exception e) { @@ -289,43 +318,50 @@ public class AlarmGroupReduce implements DisposableBean { if (shouldSendGroupImmediately(cache)) { sendGroupAlert(cache); - cache.setLastSendTime(System.currentTimeMillis()); - cache.getAlertFingerprints().clear(); } } private void sendGroupAlert(GroupAlertCache cache) { - if (cache.getAlertFingerprints().isEmpty()) { - return; - } - - long now = System.currentTimeMillis(); - String status = determineGroupStatus(cache.getAlertFingerprints().values()); - - // For firing alerts, check repeat interval - if (CommonConstants.ALERT_STATUS_FIRING.equals(status)) { - AlertGroupConverge ruleConfig = groupDefines.get(cache.getGroupDefineName()); - long repeatInterval = ruleConfig.getRepeatInterval() != null - ? ruleConfig.getRepeatInterval() * MS_PER_SECOND : DEFAULT_REPEAT_INTERVAL; - - // Skip if within repeat interval - if (cache.getLastRepeatTime() > 0 - && now - cache.getLastRepeatTime() < repeatInterval) { + synchronized (cache) { + Map snapshot = new HashMap<>(cache.getAlertFingerprints()); + if (snapshot.isEmpty()) { return; } - cache.setLastRepeatTime(now); + + long now = System.currentTimeMillis(); + String status = determineGroupStatus(snapshot.values()); + + // For firing alerts, check repeat interval without consuming the retained snapshot. + if (CommonConstants.ALERT_STATUS_FIRING.equals(status)) { + AlertGroupConverge ruleConfig = groupDefines.get(cache.getGroupDefineName()); + long repeatInterval = ruleConfig.getRepeatInterval() != null + ? ruleConfig.getRepeatInterval() * MS_PER_SECOND : DEFAULT_REPEAT_INTERVAL; + + if (cache.getLastRepeatTime() > 0 + && now - cache.getLastRepeatTime() < repeatInterval) { + return; + } + } + + GroupAlert groupAlert = GroupAlert.builder() + .groupKey(cache.getGroupKey()) + .groupLabels(cache.getGroupLabels()) + .commonLabels(extractCommonLabels(snapshot.values())) + .commonAnnotations(extractCommonAnnotations(snapshot.values())) + .alerts(new ArrayList<>(snapshot.values())) + .status(status) + .build(); + + if (!alarmInhibitReduce.inhibitAlarm(groupAlert)) { + return; + } + snapshot.forEach((fingerprint, alert) -> + cache.getAlertFingerprints().remove(fingerprint, alert)); + cache.setLastSendTime(now); + if (CommonConstants.ALERT_STATUS_FIRING.equals(status)) { + cache.setLastRepeatTime(now); + } } - - GroupAlert groupAlert = GroupAlert.builder() - .groupKey(cache.getGroupKey()) - .groupLabels(cache.getGroupLabels()) - .commonLabels(extractCommonLabels(cache.getAlertFingerprints().values())) - .commonAnnotations(extractCommonAnnotations(cache.getAlertFingerprints().values())) - .alerts(new ArrayList<>(cache.getAlertFingerprints().values())) - .status(status) - .build(); - - alarmInhibitReduce.inhibitAlarm(groupAlert); } private boolean shouldSendGroup(GroupAlertCache cache, long now) { @@ -429,6 +465,10 @@ public class AlarmGroupReduce implements DisposableBean { private boolean cancelled; + private boolean paused; + + private boolean missedWhilePaused; + private ScheduledDispatchTask(ExecutorService executor, Runnable task) { this.executor = executor; this.task = task; @@ -440,6 +480,10 @@ public class AlarmGroupReduce implements DisposableBean { if (cancelled) { return; } + if (paused) { + missedWhilePaused = true; + return; + } pendingRuns++; shouldSchedule = !running; if (shouldSchedule) { @@ -473,12 +517,14 @@ public class AlarmGroupReduce implements DisposableBean { if (cancelled) { pendingRuns = 0; running = false; + notifyAll(); return; } pendingRuns = Math.max(0, pendingRuns - 1); shouldSchedule = pendingRuns > 0; if (!shouldSchedule) { running = false; + notifyAll(); return; } } @@ -488,6 +534,49 @@ public class AlarmGroupReduce implements DisposableBean { private synchronized void cancel() { cancelled = true; pendingRuns = 0; + missedWhilePaused = false; + notifyAll(); + } + + private synchronized void pauseAdmission() { + paused = true; + missedWhilePaused |= pendingRuns > 1; + pendingRuns = running ? 1 : 0; + } + + private synchronized void awaitDrained(long timeoutNanos) + throws InterruptedException, TimeoutException { + long remainingNanos = timeoutNanos; + long startedNanos = System.nanoTime(); + while (running) { + if (remainingNanos <= 0) { + throw new TimeoutException(); + } + TimeUnit.NANOSECONDS.timedWait(this, remainingNanos); + long elapsedNanos = System.nanoTime() - startedNanos; + if (elapsedNanos <= 0) { + remainingNanos = timeoutNanos; + } else if (elapsedNanos >= timeoutNanos) { + remainingNanos = 0; + } else { + remainingNanos = timeoutNanos - elapsedNanos; + } + } + } + + private void resumeAdmission() { + boolean dispatchMissed; + synchronized (this) { + if (!paused) { + return; + } + paused = false; + dispatchMissed = missedWhilePaused && !cancelled; + missedWhilePaused = false; + } + if (dispatchMissed) { + dispatch(); + } } } } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java index a300784d1b..39747c7d48 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java @@ -203,15 +203,14 @@ public class AlarmInhibitReduce implements DisposableBean { * If alert is inhibited, it will not be forwarded * @param groupAlert Grouped and pending alerts to be processed */ - public void inhibitAlarm(GroupAlert groupAlert) { + public boolean inhibitAlarm(GroupAlert groupAlert) { if (groupAlert == null) { log.warn("Received null GroupAlert. Skipping processing."); - return; + return false; } try { if (inhibitRules.isEmpty()) { - alarmSilenceReduce.silenceAlarm(groupAlert); - return; + return alarmSilenceReduce.silenceAlarm(groupAlert); } // Process each individual alert @@ -228,10 +227,12 @@ public class AlarmInhibitReduce implements DisposableBean { // Continue processing if there are remaining alerts if (!groupAlert.getAlerts().isEmpty()) { - alarmSilenceReduce.silenceAlarm(groupAlert); + return alarmSilenceReduce.silenceAlarm(groupAlert); } + return true; } catch (Exception e) { - log.error("Error inhibiting alarm for {}", groupAlert, e); + log.error("Alarm inhibit metadata processing failed"); + return false; } } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmSilenceReduce.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmSilenceReduce.java index d903cdf7f2..301fa266fe 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmSilenceReduce.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmSilenceReduce.java @@ -46,7 +46,7 @@ public class AlarmSilenceReduce { * If alert matches any active silence rule, it will be silenced * @param groupAlert The alert to be processed */ - public void silenceAlarm(GroupAlert groupAlert) { + public boolean silenceAlarm(GroupAlert groupAlert) { List alertSilenceList = CacheFactory.getAlertSilenceCache(); if (alertSilenceList == null) { alertSilenceList = alertSilenceDao.findAlertSilencesByEnableTrue(); @@ -72,21 +72,21 @@ public class AlarmSilenceReduce { continue; } // Alert is silenced - return; + return true; } else if (alertSilence.getType() == 1) { // Cyclic silence rule int currentDayOfWeek = now.getDayOfWeek().getValue(); if (alertSilence.getDays() != null && alertSilence.getDays().contains((byte) currentDayOfWeek) && !checkAndSave(now, alertSilence)) { // Alert is silenced - return; + return true; } } } } // No matching silence rule, forward the alert - dispatcherAlarm.dispatchAlarm(groupAlert); + return dispatcherAlarm.dispatchAlarm(groupAlert); } /** diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java index d47399f1a2..709ccf3ba8 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java @@ -21,6 +21,7 @@ import static org.apache.hertzbeat.common.constants.CommonConstants.METRIC_ALERT import static org.apache.hertzbeat.common.constants.CommonConstants.TRACE_ALERT_THRESHOLD_TYPE_PERIODIC; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.clearInvocations; @@ -30,6 +31,7 @@ import static org.mockito.Mockito.when; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.apache.hertzbeat.alert.dao.AlertDefineDao; @@ -258,6 +260,76 @@ class PeriodicAlertRuleSchedulerTest { assertEquals(1, invocations.get()); } + @Test + void pauseDrainsAnEnteredPeriodicCalculationWithoutStoppingScheduler() throws Exception { + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch resumed = new CountDownLatch(1); + AtomicInteger invocations = new AtomicInteger(); + doAnswer(invocation -> { + if (invocations.incrementAndGet() == 1) { + started.countDown(); + release.await(); + } else { + resumed.countDown(); + } + return null; + }).when(metricsCalculator).calculate(any(AlertDefine.class)); + scheduler.updateSchedule(metricRule(9L)); + assertTrue(started.await(5, TimeUnit.SECONDS)); + + scheduler.pauseAdmission(); + assertThrows(TimeoutException.class, () -> scheduler.awaitDrained(0)); + release.countDown(); + scheduler.awaitDrained(TimeUnit.SECONDS.toNanos(1)); + + scheduler.resumeAdmission(); + scheduler.updateSchedule(metricRule(10L)); + assertTrue(resumed.await(5, TimeUnit.SECONDS)); + } + + @Test + void pausedTicksCoalescePerRuleAndResumeOnceWithVirtualExecutor() throws Exception { + assertPausedTicksResumeOnce(periodicProperties(true, 1)); + } + + @Test + void pausedTicksCoalescePerRuleAndResumeOnceWithScheduledExecutor() throws Exception { + assertPausedTicksResumeOnce(periodicProperties(false, 1)); + } + + private void assertPausedTicksResumeOnce(VirtualThreadProperties properties) throws Exception { + scheduler.stop(); + CountDownLatch pausedTicks = new CountDownLatch(2); + scheduler = new PeriodicAlertRuleScheduler( + metricsCalculator, logCalculator, traceCalculator, alertDefineDao, properties) { + @Override + void beforeRuleTrigger(AlertDefine rule) { + pausedTicks.countDown(); + } + }; + scheduler.start(); + CountDownLatch calculated = new CountDownLatch(1); + AtomicInteger invocations = new AtomicInteger(); + doAnswer(invocation -> { + invocations.incrementAndGet(); + calculated.countDown(); + return null; + }).when(metricsCalculator).calculate(any(AlertDefine.class)); + AlertDefine rule = metricRule(11L); + + scheduler.pauseAdmission(); + scheduler.updateSchedule(rule); + assertTrue(pausedTicks.await(3, TimeUnit.SECONDS)); + scheduler.awaitDrained(0); + + scheduler.resumeAdmission(); + scheduler.resumeAdmission(); + assertTrue(calculated.await(1, TimeUnit.SECONDS)); + scheduler.cancelSchedule(rule.getId()); + assertEquals(1, invocations.get()); + } + private AlertDefine metricRule(Long id) { return AlertDefine.builder() .id(id) @@ -279,8 +351,12 @@ class PeriodicAlertRuleSchedulerTest { } private VirtualThreadProperties periodicProperties(int maxConcurrentJobs) { + return periodicProperties(true, maxConcurrentJobs); + } + + private VirtualThreadProperties periodicProperties(boolean enabled, int maxConcurrentJobs) { return new VirtualThreadProperties( - true, + enabled, VirtualThreadProperties.PoolProperties.collectorDefaults(), VirtualThreadProperties.PoolProperties.commonDefaults(), VirtualThreadProperties.PoolProperties.managerDefaults(), diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/MetricsRealTimeAlertCalculatorMatchTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/MetricsRealTimeAlertCalculatorMatchTest.java index 7f4a61f310..5e0545925a 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/MetricsRealTimeAlertCalculatorMatchTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/MetricsRealTimeAlertCalculatorMatchTest.java @@ -23,7 +23,9 @@ import org.apache.hertzbeat.alert.calculate.AlarmCacheManager; import org.apache.hertzbeat.alert.calculate.JexlExprCalculator; import org.apache.hertzbeat.alert.dao.SingleAlertDao; import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce; +import org.apache.hertzbeat.alert.reduce.AlarmGroupReduce; import org.apache.hertzbeat.alert.service.AlertDefineService; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.constants.MetricDataConstants; import org.apache.hertzbeat.common.entity.alerter.AlertDefine; @@ -40,8 +42,12 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -71,6 +77,62 @@ public class MetricsRealTimeAlertCalculatorMatchTest { private MetricsRealTimeAlertCalculator metricsRealTimeAlertCalculator; + @Test + void positiveCapacityMaintenanceBufferKeepsTelemetryLoopForwardingLaterSamples() throws Exception { + int sampleCount = 8; + CountDownLatch stored = new CountDownLatch(sampleCount); + InMemoryCommonDataQueue queue = new InMemoryCommonDataQueue() { + @Override + public void sendMetricsDataToStorage(CollectRep.MetricsData metricsData) { + super.sendMetricsDataToStorage(metricsData); + stored.countDown(); + } + }; + AlarmCommonReduce reduce = new AlarmCommonReduce( + org.mockito.Mockito.mock(AlarmGroupReduce.class), positiveReduceCapacityProperties()); + AlerterWorkerPool loopPool = new AlerterWorkerPool(); + MetricsRealTimeAlertCalculator calculator = new MetricsRealTimeAlertCalculator( + loopPool, queue, alertDefineService, singleAlertDao, reduce, alarmCacheManager, + new JexlExprCalculator(), false) { + @Override + protected void calculate(CollectRep.MetricsData metricsData) { + reduce.reduceAndSendAlarm(org.apache.hertzbeat.common.entity.alerter.SingleAlert.builder() + .labels(Map.of("sample", Long.toString(metricsData.getId()))) + .build()); + } + }; + reduce.pauseAdmission(); + calculator.startCalculate(); + for (int index = 0; index < sampleCount; index++) { + queue.sendMetricsData(CollectRep.MetricsData.newBuilder().setId(index + 1L).build()); + } + + assertTrue(stored.await(2, TimeUnit.SECONDS)); + for (int index = 0; index < sampleCount; index++) { + assertNotNull(queue.pollMetricsDataToStorage()); + } + + reduce.destroy(); + loopPool.destroy(); + } + + private static VirtualThreadProperties positiveReduceCapacityProperties() { + return new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + new VirtualThreadProperties.AlerterProperties( + VirtualThreadProperties.PoolProperties.alerterNotifyDefaults(), + 10, + VirtualThreadProperties.QueueProperties.logWorkerDefaults(), + new VirtualThreadProperties.QueueProperties(1, 1), + VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(), + 4), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); + } + @BeforeEach public void setUp() { MockitoAnnotations.openMocks(this); @@ -169,6 +231,7 @@ public class MetricsRealTimeAlertCalculatorMatchTest { verify(alarmCacheManager, times(1)).getPending(any(), any()); verify(alarmCacheManager, times(1)).putFiring(any(), any(), any()); verify(alarmCommonReduce, times(1)).reduceAndSendAlarm(any()); + verify(dataQueue, times(1)).sendMetricsDataToStorage(metricsData); } @Test diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatchTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatchTest.java index b025d931b2..8c599e2974 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatchTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatchTest.java @@ -24,6 +24,8 @@ import static org.mockito.ArgumentMatchers.anyByte; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -180,4 +182,18 @@ class AlertNoticeDispatchTest { verify(alertNotifyHandler).send(eq(receiver), eq(template), eq(alert)); verify(emitterManager).broadcast(any(String.class)); } + + @Test + void postStoreFailureDoesNotChangeMetadataSuccessOutcome() { + when(alertStoreHandler.store(alert)).thenReturn(alert); + when(noticeConfigService.getReceiverFilterRule(alert)) + .thenThrow(new IllegalStateException("notice unavailable")); + doThrow(new IllegalStateException("broadcast unavailable")) + .when(emitterManager).broadcast(any(String.class)); + + assertTrue(alertNoticeDispatch.dispatchAlarm(alert)); + + verify(alertStoreHandler, times(1)).store(alert); + verify(emitterManager).broadcast(any(String.class)); + } } diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduceTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduceTest.java index 0208188529..bc9d25ac41 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduceTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduceTest.java @@ -18,18 +18,25 @@ package org.apache.hertzbeat.alert.reduce; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import java.util.HashMap; import java.util.Map; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -135,4 +142,208 @@ class AlarmCommonReduceTest { assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); } + @Test + void pauseDefersNewReducersAndReplaysEachOnceAfterDrain() throws Exception { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch resumedStarted = new CountDownLatch(1); + AtomicInteger invocations = new AtomicInteger(); + doAnswer(invocation -> { + int current = invocations.incrementAndGet(); + if (current == 1) { + firstStarted.countDown(); + releaseFirst.await(); + } else { + resumedStarted.countDown(); + } + return null; + }).when(alarmGroupReduce).processGroupAlert(any(SingleAlert.class)); + + alarmCommonReduce.reduceAndSendAlarm(testAlert); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + alarmCommonReduce.pauseAdmission(); + assertThrows(TimeoutException.class, () -> alarmCommonReduce.awaitDrained(0)); + alarmCommonReduce.reduceAndSendAlarm(testAlert); + releaseFirst.countDown(); + alarmCommonReduce.awaitDrained(TimeUnit.SECONDS.toNanos(1)); + verify(alarmGroupReduce, times(1)).processGroupAlert(any(SingleAlert.class)); + + alarmCommonReduce.resumeAdmission(); + assertTrue(resumedStarted.await(5, TimeUnit.SECONDS)); + verify(alarmGroupReduce, times(2)).processGroupAlert(any(SingleAlert.class)); + } + + @Test + void pauseCannotLetDeferredWaiterOvertakeReservedSubmission() throws Exception { + CountDownLatch permitReserved = new CountDownLatch(1); + CountDownLatch releaseSubmission = new CountDownLatch(1); + alarmCommonReduce.destroy(); + alarmCommonReduce = new TestAlarmCommonReduce( + alarmGroupReduce, permitReserved, releaseSubmission); + CountDownLatch firstRunning = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondRunning = new CountDownLatch(1); + AtomicInteger invocations = new AtomicInteger(); + doAnswer(invocation -> { + if (invocations.incrementAndGet() == 1) { + firstRunning.countDown(); + releaseFirst.await(); + } else { + secondRunning.countDown(); + } + return null; + }).when(alarmGroupReduce).processGroupAlert(any(SingleAlert.class)); + + Thread admitted = Thread.ofPlatform().start(() -> alarmCommonReduce.reduceAndSendAlarm(testAlert)); + assertTrue(permitReserved.await(1, TimeUnit.SECONDS)); + Thread pause = Thread.ofPlatform().start(alarmCommonReduce::pauseAdmission); + while (!alarmCommonReduce.hasQueuedMaintenanceThread(pause)) { + Thread.onSpinWait(); + } + Thread deferred = Thread.ofPlatform().start(() -> alarmCommonReduce.reduceAndSendAlarm(testAlert)); + releaseSubmission.countDown(); + admitted.join(1_000); + pause.join(1_000); + deferred.join(1_000); + assertTrue(firstRunning.await(1, TimeUnit.SECONDS)); + + assertThrows(TimeoutException.class, () -> alarmCommonReduce.awaitDrained(0)); + releaseFirst.countDown(); + alarmCommonReduce.awaitDrained(TimeUnit.SECONDS.toNanos(1)); + verify(alarmGroupReduce, times(1)).processGroupAlert(any(SingleAlert.class)); + + alarmCommonReduce.resumeAdmission(); + assertTrue(secondRunning.await(1, TimeUnit.SECONDS)); + verify(alarmGroupReduce, times(2)).processGroupAlert(any(SingleAlert.class)); + } + + @Test + void maintenanceDeferralNeverBlocksProducerAtWorkerQueueCapacity() throws Exception { + ManagedExecutor executor = org.mockito.Mockito.mock(ManagedExecutor.class); + org.mockito.Mockito.doAnswer(invocation -> { + ((Runnable) invocation.getArgument(0)).run(); + return null; + }).when(executor).execute(any(Runnable.class)); + alarmCommonReduce.destroy(); + alarmCommonReduce = new AlarmCommonReduce(alarmGroupReduce, executor); + alarmCommonReduce.pauseAdmission(); + alarmCommonReduce.reduceAndSendAlarm(testAlert); + CountDownLatch callerContinued = new CountDownLatch(1); + Thread second = Thread.ofPlatform().start(() -> { + alarmCommonReduce.reduceAndSendAlarm(testAlert); + callerContinued.countDown(); + }); + assertTrue(callerContinued.await(1, TimeUnit.SECONDS)); + second.join(1_000); + + verify(executor, times(0)).execute(any(Runnable.class)); + assertEquals(2, alarmCommonReduce.deferredTaskCount()); + + alarmCommonReduce.resumeAdmission(); + verify(alarmGroupReduce, times(2)).processGroupAlert(any(SingleAlert.class)); + } + + @Test + void replayQueueRejectionRetainsEveryUnsubmittedDeferredTask() { + ManagedExecutor executor = org.mockito.Mockito.mock(ManagedExecutor.class); + AtomicInteger submissions = new AtomicInteger(); + org.mockito.Mockito.doAnswer(invocation -> { + if (submissions.incrementAndGet() == 2) { + throw new RejectedExecutionException(); + } + ((Runnable) invocation.getArgument(0)).run(); + return null; + }).when(executor).execute(any(Runnable.class)); + alarmCommonReduce.destroy(); + alarmCommonReduce = new AlarmCommonReduce(alarmGroupReduce, executor); + alarmCommonReduce.pauseAdmission(); + alarmCommonReduce.reduceAndSendAlarm(testAlert); + alarmCommonReduce.reduceAndSendAlarm(testAlert); + + assertThrows(RejectedExecutionException.class, alarmCommonReduce::resumeAdmission); + assertEquals(1, alarmCommonReduce.deferredTaskCount()); + + org.mockito.Mockito.doAnswer(invocation -> { + ((Runnable) invocation.getArgument(0)).run(); + return null; + }).when(executor).execute(any(Runnable.class)); + alarmCommonReduce.resumeAdmission(); + + assertEquals(0, alarmCommonReduce.deferredTaskCount()); + verify(alarmGroupReduce, times(2)).processGroupAlert(any(SingleAlert.class)); + } + + @Test + void rejectedReplayRemainsDeferredForOneRetry() { + ManagedExecutor executor = org.mockito.Mockito.mock(ManagedExecutor.class); + org.mockito.Mockito.doThrow(new RejectedExecutionException()).when(executor).execute(any(Runnable.class)); + alarmCommonReduce.destroy(); + alarmCommonReduce = new AlarmCommonReduce(alarmGroupReduce, executor); + alarmCommonReduce.pauseAdmission(); + alarmCommonReduce.reduceAndSendAlarm(testAlert); + + assertThrows(RejectedExecutionException.class, + alarmCommonReduce::resumeAdmission); + assertEquals(1, alarmCommonReduce.deferredTaskCount()); + + org.mockito.Mockito.doAnswer(invocation -> { + ((Runnable) invocation.getArgument(0)).run(); + return null; + }).when(executor).execute(any(Runnable.class)); + alarmCommonReduce.resumeAdmission(); + + assertEquals(0, alarmCommonReduce.deferredTaskCount()); + verify(alarmGroupReduce, times(1)).processGroupAlert(any(SingleAlert.class)); + } + + private static final class TestAlarmCommonReduce extends AlarmCommonReduce { + + private final CountDownLatch permitReserved; + private final CountDownLatch releaseSubmission; + private final AtomicBoolean first = new AtomicBoolean(true); + + private TestAlarmCommonReduce( + AlarmGroupReduce alarmGroupReduce, + CountDownLatch permitReserved, + CountDownLatch releaseSubmission) { + super(alarmGroupReduce, singleWorkerProperties()); + this.permitReserved = permitReserved; + this.releaseSubmission = releaseSubmission; + } + + @Override + void beforeAdmittedSubmission() { + if (first.compareAndSet(true, false)) { + permitReserved.countDown(); + try { + releaseSubmission.await(); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + } + } + } + } + + private static VirtualThreadProperties singleWorkerProperties() { + return singleWorkerProperties(1); + } + + private static VirtualThreadProperties singleWorkerProperties(int queueCapacity) { + return new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + new VirtualThreadProperties.AlerterProperties( + VirtualThreadProperties.PoolProperties.alerterNotifyDefaults(), + 10, + VirtualThreadProperties.QueueProperties.logWorkerDefaults(), + new VirtualThreadProperties.QueueProperties(1, queueCapacity), + VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(), + 4), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); + } + } diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java index 9007b9c3f7..0b6870c06f 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java @@ -40,11 +40,13 @@ package org.apache.hertzbeat.alert.reduce; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.never; import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; @@ -52,9 +54,11 @@ import static org.mockito.Mockito.when; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.apache.hertzbeat.alert.dao.AlertGroupConvergeDao; @@ -98,6 +102,7 @@ class AlarmGroupReduceTest { @BeforeEach void setUp() { MockitoAnnotations.openMocks(this); + when(alarmInhibitReduce.inhibitAlarm(any())).thenReturn(true); when(alertGroupConvergeDao.findAlertGroupConvergesByEnableIsTrue()) .thenReturn(Collections.emptyList()); alarmGroupReduce = new AlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao, @@ -216,6 +221,102 @@ class AlarmGroupReduceTest { assertEquals(1, invocations.get()); } + @Test + void pauseDrainsRunningGroupPassAndCoalescesOneMissedPass() throws Exception { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch resumedStarted = new CountDownLatch(1); + AtomicInteger invocations = new AtomicInteger(); + alarmGroupReduce.destroy(); + alarmGroupReduce = new TestAlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao, + new VirtualThreadProperties(), null, null, firstStarted, releaseFirst, resumedStarted, + new AtomicInteger(), invocations); + alarmGroupReduce.start(); + alarmGroupReduce.dispatchCheckAndSendGroups(); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + alarmGroupReduce.pauseAdmission(); + assertThrows(TimeoutException.class, () -> alarmGroupReduce.awaitDrained(0)); + alarmGroupReduce.dispatchCheckAndSendGroups(); + releaseFirst.countDown(); + alarmGroupReduce.awaitDrained(TimeUnit.SECONDS.toNanos(1)); + + alarmGroupReduce.resumeAdmission(); + assertTrue(resumedStarted.await(5, TimeUnit.SECONDS)); + assertEquals(2, invocations.get()); + } + + @Test + void failedGroupStoreRetainsSnapshotForOneRetry() throws Exception { + alarmGroupReduce.refreshGroupDefines(List.of(groupRule(0))); + when(alarmInhibitReduce.inhibitAlarm(any())).thenReturn(false, true); + alarmGroupReduce.processGroupAlert(groupAlert("fp-1", "firing")); + + dispatchAndDrain(); + dispatchAndDrain(); + + verify(alarmInhibitReduce, times(2)).inhibitAlarm(any()); + } + + @Test + void repeatSkipRetainsFiringUntilResolvedSnapshotIsStored() throws Exception { + AlertGroupConverge rule = groupRule(600); + alarmGroupReduce.refreshGroupDefines(List.of(rule)); + alarmGroupReduce.processGroupAlert(groupAlert("fp-1", "firing")); + dispatchAndDrain(); + alarmGroupReduce.processGroupAlert(groupAlert("fp-1", "firing")); + dispatchAndDrain(); + alarmGroupReduce.processGroupAlert(groupAlert("fp-1", "resolved")); + dispatchAndDrain(); + + verify(alarmInhibitReduce, times(2)).inhibitAlarm(any()); + } + + @Test + void concurrentInsertIsNotClearedWithSuccessfulSnapshot() throws Exception { + alarmGroupReduce.refreshGroupDefines(List.of(groupRule(0))); + AtomicBoolean inserted = new AtomicBoolean(); + doAnswer(invocation -> { + if (inserted.compareAndSet(false, true)) { + alarmGroupReduce.processGroupAlert(groupAlert("fp-2", "firing")); + } + return true; + }).when(alarmInhibitReduce).inhibitAlarm(any()); + alarmGroupReduce.processGroupAlert(groupAlert("fp-1", "firing")); + + dispatchAndDrain(); + dispatchAndDrain(); + dispatchAndDrain(); + + verify(alarmInhibitReduce, times(2)).inhibitAlarm(any()); + } + + private void dispatchAndDrain() throws Exception { + alarmGroupReduce.dispatchCheckAndSendGroups(); + alarmGroupReduce.pauseAdmission(); + alarmGroupReduce.awaitDrained(TimeUnit.SECONDS.toNanos(1)); + alarmGroupReduce.resumeAdmission(); + } + + private AlertGroupConverge groupRule(long repeatInterval) { + AlertGroupConverge rule = new AlertGroupConverge(); + rule.setName("test-rule"); + rule.setGroupLabels(List.of("severity")); + rule.setGroupWait(0L); + rule.setGroupInterval(0L); + rule.setRepeatInterval(repeatInterval); + return rule; + } + + private SingleAlert groupAlert(String fingerprint, String status) { + return SingleAlert.builder() + .fingerprint(fingerprint) + .status(status) + .labels(createLabels("severity", "critical")) + .annotations(Map.of()) + .build(); + } + private Map createLabels(String... keyValues) { Map labels = new HashMap<>(); for (int i = 0; i < keyValues.length; i += 2) { diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduceTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduceTest.java index 9324c5fdcd..db85633d93 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduceTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduceTest.java @@ -42,6 +42,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -233,6 +234,15 @@ class AlarmInhibitReduceTest { verify(alarmSilenceReduce).silenceAlarm(alert); } + @Test + void synchronousSilenceOrStoreFailureIsReportedToGroupOwner() { + GroupAlert alert = GroupAlert.builder().alerts(new ArrayList<>()).build(); + doThrow(new IllegalStateException("store unavailable")) + .when(alarmSilenceReduce).silenceAlarm(alert); + + assertFalse(alarmInhibitReduce.inhibitAlarm(alert)); + } + @Test void whenMultipleSourceAlerts_shouldInhibitAllMatchingTargets() { AlertInhibit rule = AlertInhibit.builder() diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/WorkAdmissionGate.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/WorkAdmissionGate.java new file mode 100644 index 0000000000..b2b2e34e47 --- /dev/null +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/WorkAdmissionGate.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.common.concurrent; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * Process-local admission gate for work that must drain without owning its executor. + */ +public final class WorkAdmissionGate { + + private final Object lock = new Object(); + private boolean accepting = true; + private boolean stopped; + private int activeWork; + private int waitingWork; + + /** + * Returns a permit for admitted work, or {@code null} while admission is paused. + */ + public Permit tryAcquire() { + synchronized (lock) { + if (!accepting) { + return null; + } + activeWork++; + return new Permit(this); + } + } + + /** + * Reserves replay work while ordinary admission is still paused. + */ + public Permit reserveReplay() { + synchronized (lock) { + if (stopped) { + return null; + } + activeWork++; + return new Permit(this); + } + } + + /** + * Waits for resumed admission, or returns {@code null} after terminal stop. + */ + public Permit awaitAcquire() throws InterruptedException { + synchronized (lock) { + while (!accepting && !stopped) { + waitingWork++; + try { + lock.wait(); + } finally { + waitingWork--; + } + } + if (stopped) { + return null; + } + activeWork++; + return new Permit(this); + } + } + + public void pauseAdmission() { + synchronized (lock) { + accepting = false; + } + } + + public void awaitDrained(long timeoutNanos) throws InterruptedException, TimeoutException { + synchronized (lock) { + long remainingNanos = timeoutNanos; + long startedNanos = System.nanoTime(); + while (activeWork > 0) { + if (remainingNanos <= 0) { + throw new TimeoutException(); + } + TimeUnit.NANOSECONDS.timedWait(lock, remainingNanos); + long elapsedNanos = System.nanoTime() - startedNanos; + if (elapsedNanos <= 0) { + remainingNanos = timeoutNanos; + } else if (elapsedNanos >= timeoutNanos) { + remainingNanos = 0; + } else { + remainingNanos = timeoutNanos - elapsedNanos; + } + } + } + } + + public void resumeAdmission() { + synchronized (lock) { + if (!stopped) { + accepting = true; + } + lock.notifyAll(); + } + } + + /** + * Permanently rejects admission and wakes work waiting for a maintenance resume. + */ + public void stop() { + synchronized (lock) { + stopped = true; + accepting = false; + lock.notifyAll(); + } + } + + int waitingWork() { + synchronized (lock) { + return waitingWork; + } + } + + private void release() { + synchronized (lock) { + if (activeWork > 0) { + activeWork--; + lock.notifyAll(); + } + } + } + + /** + * Idempotent ownership token for one admitted unit of work. + */ + public static final class Permit implements AutoCloseable { + + private final WorkAdmissionGate gate; + private boolean closed; + + private Permit(WorkAdmissionGate gate) { + this.gate = gate; + } + + @Override + public synchronized void close() { + if (!closed) { + closed = true; + gate.release(); + } + } + } +} diff --git a/hertzbeat-common-core/src/test/java/org/apache/hertzbeat/common/concurrent/WorkAdmissionGateTest.java b/hertzbeat-common-core/src/test/java/org/apache/hertzbeat/common/concurrent/WorkAdmissionGateTest.java new file mode 100644 index 0000000000..ad0f18c0d6 --- /dev/null +++ b/hertzbeat-common-core/src/test/java/org/apache/hertzbeat/common/concurrent/WorkAdmissionGateTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.common.concurrent; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; + +class WorkAdmissionGateTest { + + @Test + void pauseRejectsNewWorkAndDrainsOnlyAdmittedWork() throws Exception { + WorkAdmissionGate gate = new WorkAdmissionGate(); + WorkAdmissionGate.Permit admitted = gate.tryAcquire(); + + gate.pauseAdmission(); + + assertThat(gate.tryAcquire()).isNull(); + assertThatThrownBy(() -> gate.awaitDrained(0)) + .isInstanceOf(TimeoutException.class); + + admitted.close(); + gate.awaitDrained(TimeUnit.SECONDS.toNanos(1)); + gate.resumeAdmission(); + + assertThat(gate.tryAcquire()).isNotNull().satisfies(WorkAdmissionGate.Permit::close); + } + + @Test + void repeatedPauseResumeDoesNotDuplicatePermits() throws Exception { + WorkAdmissionGate gate = new WorkAdmissionGate(); + + gate.pauseAdmission(); + gate.pauseAdmission(); + gate.awaitDrained(0); + gate.resumeAdmission(); + gate.resumeAdmission(); + + WorkAdmissionGate.Permit permit = gate.tryAcquire(); + assertThat(permit).isNotNull(); + permit.close(); + gate.awaitDrained(0); + } + + @Test + void waitingAdmissionResumesOnceAndInterruptDoesNotLeakPermit() throws Exception { + WorkAdmissionGate gate = new WorkAdmissionGate(); + gate.pauseAdmission(); + AtomicReference resumed = new AtomicReference<>(); + Thread waiter = Thread.ofPlatform().start(() -> { + try { + resumed.set(gate.awaitAcquire()); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + } + }); + while (gate.waitingWork() == 0) { + Thread.onSpinWait(); + } + + gate.resumeAdmission(); + waiter.join(1_000); + assertThat(waiter.isAlive()).isFalse(); + assertThat(resumed.get()).isNotNull(); + resumed.get().close(); + gate.awaitDrained(0); + + gate.pauseAdmission(); + AtomicBoolean interrupted = new AtomicBoolean(); + Thread interruptedWaiter = Thread.ofPlatform().start(() -> { + try { + gate.awaitAcquire(); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + interrupted.set(Thread.currentThread().isInterrupted()); + } + }); + while (gate.waitingWork() == 0) { + Thread.onSpinWait(); + } + interruptedWaiter.interrupt(); + interruptedWaiter.join(1_000); + + assertThat(interrupted.get()).isTrue(); + gate.awaitDrained(0); + } + + @Test + void terminalStopWakesWaiterAndResumeCannotReviveAdmission() throws Exception { + WorkAdmissionGate gate = new WorkAdmissionGate(); + gate.pauseAdmission(); + AtomicReference result = new AtomicReference<>(); + Thread waiter = Thread.ofPlatform().start(() -> { + try { + result.set(gate.awaitAcquire()); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + } + }); + while (gate.waitingWork() == 0) { + Thread.onSpinWait(); + } + + gate.stop(); + gate.resumeAdmission(); + waiter.join(1_000); + + assertThat(waiter.isAlive()).isFalse(); + assertThat(result.get()).isNull(); + assertThat(gate.tryAcquire()).isNull(); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/AlertMetadataMaintenanceParticipant.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/AlertMetadataMaintenanceParticipant.java new file mode 100644 index 0000000000..65f14948b9 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/AlertMetadataMaintenanceParticipant.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import java.time.Duration; +import java.util.concurrent.TimeoutException; +import org.apache.hertzbeat.alert.calculate.periodic.PeriodicAlertRuleScheduler; +import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce; +import org.apache.hertzbeat.alert.reduce.AlarmGroupReduce; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; + +/** Sequential cut across alert producers whose downstream path writes management metadata. */ +@Component +@ConditionalOnNormalBusinessRuntime +@Order(400) +public final class AlertMetadataMaintenanceParticipant implements MetadataMaintenanceParticipant { + + private final PeriodicAlertRuleScheduler periodicScheduler; + private final AlarmCommonReduce commonReduce; + private final AlarmGroupReduce groupReduce; + private MetadataMaintenancePhase phase = MetadataMaintenancePhase.RUNNING; + private boolean periodicPaused; + private boolean commonPaused; + private boolean groupPaused; + + public AlertMetadataMaintenanceParticipant( + PeriodicAlertRuleScheduler periodicScheduler, + AlarmCommonReduce commonReduce, + AlarmGroupReduce groupReduce) { + this.periodicScheduler = periodicScheduler; + this.commonReduce = commonReduce; + this.groupReduce = groupReduce; + } + + @Override + public String participantId() { + return "alert-control-metadata"; + } + + @Override + public synchronized void quiesce(Duration timeout) { + if (phase == MetadataMaintenancePhase.QUIESCED) { + return; + } + MaintenanceDeadline deadline = MaintenanceDeadline.start(timeout); + phase = MetadataMaintenancePhase.QUIESCING; + try { + periodicScheduler.pauseAdmission(); + periodicPaused = true; + periodicScheduler.awaitDrained(deadline.remainingNanos()); + commonReduce.pauseAdmission(); + commonPaused = true; + commonReduce.awaitDrained(deadline.remainingNanos()); + groupReduce.pauseAdmission(); + groupPaused = true; + groupReduce.awaitDrained(deadline.remainingNanos()); + phase = MetadataMaintenancePhase.QUIESCED; + } catch (InterruptedException exception) { + resumePausedStages(); + Thread.currentThread().interrupt(); + throw MetadataMaintenanceException.quiesceInterrupted(); + } catch (TimeoutException exception) { + resumePausedStages(); + throw MetadataMaintenanceException.quiesceTimeout(); + } catch (RuntimeException exception) { + resumePausedStages(); + throw MetadataMaintenanceException.participantFailure(); + } + } + + @Override + public synchronized void resume() { + if (phase == MetadataMaintenancePhase.RUNNING) { + return; + } + if (!resumePausedStages()) { + throw MetadataMaintenanceException.resumeFailure(); + } + } + + private boolean resumePausedStages() { + boolean resumed = true; + if (groupPaused) { + try { + groupReduce.resumeAdmission(); + groupPaused = false; + } catch (RuntimeException exception) { + resumed = false; + } + } + if (commonPaused) { + try { + commonReduce.resumeAdmission(); + commonPaused = false; + } catch (RuntimeException exception) { + resumed = false; + } + } + if (periodicPaused) { + try { + periodicScheduler.resumeAdmission(); + periodicPaused = false; + } catch (RuntimeException exception) { + resumed = false; + } + } + phase = resumed ? MetadataMaintenancePhase.RUNNING : MetadataMaintenancePhase.RECOVERY_REQUIRED; + return resumed; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/CollectorLifecycleMaintenanceParticipant.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/CollectorLifecycleMaintenanceParticipant.java new file mode 100644 index 0000000000..ddba74e8be --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/CollectorLifecycleMaintenanceParticipant.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import java.time.Duration; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeoutException; +import org.apache.hertzbeat.alert.calculate.CollectorAlertHandler; +import org.apache.hertzbeat.common.concurrent.WorkAdmissionGate; +import org.apache.hertzbeat.common.entity.dto.CollectorInfo; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; +import org.apache.hertzbeat.manager.scheduler.CollectorJobScheduler; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; + +/** Coalesces collector lifecycle intent while draining metadata and paired alert work. */ +@Component +@ConditionalOnNormalBusinessRuntime +@Order(300) +public final class CollectorLifecycleMaintenanceParticipant implements MetadataMaintenanceParticipant { + + private final Object lock = new Object(); + private final CollectorJobScheduler scheduler; + private final CollectorAlertHandler alertHandler; + private final WorkAdmissionGate maintenanceGate = new WorkAdmissionGate(); + private final Map pendingTransitions = new HashMap<>(); + private final Set runningIdentities = new HashSet<>(); + private MetadataMaintenancePhase phase = MetadataMaintenancePhase.RUNNING; + private long generation; + + public CollectorLifecycleMaintenanceParticipant( + CollectorJobScheduler scheduler, CollectorAlertHandler alertHandler) { + this.scheduler = scheduler; + this.alertHandler = alertHandler; + } + + @Override + public String participantId() { + return "collector-control-metadata"; + } + + public void collectorOnline(String identity, CollectorInfo collectorInfo, boolean submitAlert) { + submit(identity, true, collectorInfo, submitAlert); + } + + public void collectorOffline(String identity, boolean submitAlert) { + submit(identity, false, null, submitAlert); + } + + @Override + public void quiesce(Duration timeout) { + MaintenanceDeadline deadline = MaintenanceDeadline.start(timeout); + synchronized (lock) { + if (phase == MetadataMaintenancePhase.QUIESCED) { + return; + } + phase = MetadataMaintenancePhase.QUIESCING; + maintenanceGate.pauseAdmission(); + } + try { + maintenanceGate.awaitDrained(deadline.remainingNanos()); + synchronized (lock) { + phase = MetadataMaintenancePhase.QUIESCED; + } + } catch (InterruptedException exception) { + reopenAfterFailedQuiesce(); + Thread.currentThread().interrupt(); + throw MetadataMaintenanceException.quiesceInterrupted(); + } catch (TimeoutException exception) { + reopenAfterFailedQuiesce(); + throw MetadataMaintenanceException.quiesceTimeout(); + } + } + + @Override + public void resume() { + while (true) { + Transition transition; + synchronized (lock) { + transition = pendingTransitions.values().stream() + .min(Comparator.comparingLong(Transition::generation)) + .orElse(null); + if (transition == null) { + maintenanceGate.resumeAdmission(); + phase = MetadataMaintenancePhase.RUNNING; + return; + } + pendingTransitions.remove(transition.identity(), transition); + runningIdentities.add(transition.identity()); + } + try { + execute(transition); + } catch (RuntimeException exception) { + synchronized (lock) { + pendingTransitions.putIfAbsent(transition.identity(), transition); + runningIdentities.remove(transition.identity()); + } + throw exception; + } finally { + synchronized (lock) { + runningIdentities.remove(transition.identity()); + } + } + } + } + + private void submit(String identity, boolean online, CollectorInfo collectorInfo, boolean submitAlert) { + WorkAdmissionGate.Permit permit; + Transition transition; + synchronized (lock) { + transition = new Transition(identity, online, collectorInfo, submitAlert, ++generation); + if (runningIdentities.contains(identity)) { + pendingTransitions.put(identity, transition); + return; + } + // A failed transition is retained only until a newer observed intent supersedes it. + pendingTransitions.remove(identity); + permit = maintenanceGate.tryAcquire(); + if (permit == null) { + pendingTransitions.put(identity, transition); + return; + } + runningIdentities.add(identity); + } + executeAdmitted(transition, permit); + } + + private void executeAdmitted(Transition firstTransition, WorkAdmissionGate.Permit firstPermit) { + Transition transition = firstTransition; + WorkAdmissionGate.Permit permit = firstPermit; + RuntimeException firstFailure = null; + while (true) { + boolean transitionFailed = false; + try { + execute(transition); + } catch (RuntimeException exception) { + transitionFailed = true; + if (firstFailure == null) { + firstFailure = exception; + } else if (firstFailure.getSuppressed().length == 0) { + firstFailure.addSuppressed(exception); + } + } finally { + permit.close(); + } + synchronized (lock) { + runningIdentities.remove(transition.identity()); + Transition next = pendingTransitions.remove(transition.identity()); + if (next == null) { + if (transitionFailed) { + pendingTransitions.put(transition.identity(), transition); + } + throwIfFailed(firstFailure); + return; + } + permit = maintenanceGate.tryAcquire(); + if (permit == null) { + pendingTransitions.put(next.identity(), next); + throwIfFailed(firstFailure); + return; + } + runningIdentities.add(next.identity()); + transition = next; + } + } + } + + private void throwIfFailed(RuntimeException failure) { + if (failure != null) { + throw failure; + } + } + + private void execute(Transition transition) { + if (transition.online()) { + if (transition.submitAlert()) { + alertHandler.online(transition.identity()); + } + scheduler.collectorGoOnline(transition.identity(), transition.collectorInfo()); + return; + } + scheduler.collectorGoOffline(transition.identity()); + if (transition.submitAlert()) { + alertHandler.offline(transition.identity()); + } + } + + private void reopenAfterFailedQuiesce() { + synchronized (lock) { + maintenanceGate.resumeAdmission(); + phase = MetadataMaintenancePhase.RUNNING; + } + } + + private record Transition( + String identity, + boolean online, + CollectorInfo collectorInfo, + boolean submitAlert, + long generation) { + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceCoordinator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceCoordinator.java index 2f79338a83..de8e8f26b6 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceCoordinator.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceCoordinator.java @@ -45,20 +45,20 @@ public final class MetadataMaintenanceCoordinator { requireOperationId(requestedOperationId); Acquisition acquisition = beginAcquisition(requestedOperationId); - List completed = new ArrayList<>(participants.size()); + List started = new ArrayList<>(participants.size()); try { for (MetadataMaintenanceParticipant participant : participants) { + started.add(participant); participant.quiesce(deadline.remaining()); - completed.add(participant); } } catch (MetadataMaintenanceException exception) { - rollback(acquisition, completed); + rollback(acquisition, started); throw exception; } catch (Error error) { - rollback(acquisition, completed); + rollback(acquisition, started); throw error; } catch (RuntimeException exception) { - rollback(acquisition, completed); + rollback(acquisition, started); throw MetadataMaintenanceException.participantFailure(); } return completeAcquisition(acquisition); @@ -100,6 +100,15 @@ public final class MetadataMaintenanceCoordinator { private Acquisition beginAcquisition(String requestedOperationId) { lock.lock(); try { + if (phase == MetadataMaintenancePhase.RECOVERY_REQUIRED) { + if (!requestedOperationId.equals(operationId)) { + throw MetadataMaintenanceException.operationConflict(); + } + if (!resumeAllParticipants()) { + throw MetadataMaintenanceException.resumeFailure(); + } + reopen(); + } if (phase != MetadataMaintenancePhase.RUNNING) { throw MetadataMaintenanceException.operationConflict(); } @@ -114,6 +123,18 @@ public final class MetadataMaintenanceCoordinator { } } + private boolean resumeAllParticipants() { + boolean resumed = true; + for (int index = participants.size() - 1; index >= 0; index--) { + try { + participants.get(index).resume(); + } catch (RuntimeException exception) { + resumed = false; + } + } + return resumed; + } + private MetadataMaintenanceLease completeAcquisition(Acquisition acquisition) { lock.lock(); try { @@ -127,19 +148,22 @@ public final class MetadataMaintenanceCoordinator { } } - private void rollback(Acquisition acquisition, List completed) { - Collections.reverse(completed); - for (MetadataMaintenanceParticipant participant : completed) { + private void rollback(Acquisition acquisition, List started) { + Collections.reverse(started); + boolean failed = false; + for (MetadataMaintenanceParticipant participant : started) { try { participant.resume(); } catch (RuntimeException exception) { - // Rollback is best effort and must not replace the primary safe failure category. + failed = true; } } lock.lock(); try { - if (ownsAcquisition(acquisition)) { + if (ownsAcquisition(acquisition) && !failed) { reopen(); + } else if (ownsAcquisition(acquisition)) { + phase = MetadataMaintenancePhase.RECOVERY_REQUIRED; } } finally { lock.unlock(); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenancePhase.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenancePhase.java index e0997a4673..a20ba8e15f 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenancePhase.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenancePhase.java @@ -11,5 +11,6 @@ package org.apache.hertzbeat.manager.maintenance; public enum MetadataMaintenancePhase { RUNNING, QUIESCING, - QUIESCED + QUIESCED, + RECOVERY_REQUIRED } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServer.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServer.java index 0649e5d5ad..d7a03eb3ae 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServer.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServer.java @@ -28,9 +28,11 @@ import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.alert.calculate.CollectorAlertHandler; import org.apache.hertzbeat.common.concurrent.BackgroundTaskExecutor; import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.apache.hertzbeat.common.entity.dto.CollectorInfo; import org.apache.hertzbeat.common.entity.message.ClusterMsg; import org.apache.hertzbeat.common.queue.CommonDataQueue; import org.apache.hertzbeat.manager.scheduler.CollectorJobScheduler; +import org.apache.hertzbeat.manager.maintenance.CollectorLifecycleMaintenanceParticipant; import org.apache.hertzbeat.manager.scheduler.SchedulerProperties; import org.apache.hertzbeat.manager.scheduler.netty.process.CollectCyclicDataResponseProcessor; import org.apache.hertzbeat.manager.scheduler.netty.process.CollectCyclicServiceDiscoveryDataResponseProcessor; @@ -45,6 +47,7 @@ import org.apache.hertzbeat.remoting.event.NettyEventListener; import org.apache.hertzbeat.remoting.netty.NettyRemotingServer; import org.apache.hertzbeat.remoting.netty.NettyServerConfig; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.lang.Nullable; import org.springframework.stereotype.Component; /** @@ -64,6 +67,8 @@ public class ManageServer { private final CollectorRuntimeConfigService runtimeConfigService; + private final CollectorLifecycleMaintenanceParticipant collectorLifecycleMaintenance; + private ScheduledExecutorService channelSchedule; private ChannelCheckGeneration channelCheckGeneration; @@ -97,7 +102,7 @@ public class ManageServer { final CommonDataQueue commonDataQueue, final VirtualThreadProperties virtualThreadProperties) { this(schedulerProperties, collectorJobScheduler, threadPool, collectorAlertHandler, commonDataQueue, - virtualThreadProperties, new CollectorRuntimeStatusRegistry(), null); + virtualThreadProperties, new CollectorRuntimeStatusRegistry(), null, null); } public ManageServer(final SchedulerProperties schedulerProperties, @@ -108,7 +113,7 @@ public class ManageServer { final VirtualThreadProperties virtualThreadProperties, final CollectorRuntimeStatusRegistry runtimeStatusRegistry) { this(schedulerProperties, collectorJobScheduler, threadPool, collectorAlertHandler, commonDataQueue, - virtualThreadProperties, runtimeStatusRegistry, null); + virtualThreadProperties, runtimeStatusRegistry, null, null); } @Autowired @@ -119,13 +124,15 @@ public class ManageServer { final CommonDataQueue commonDataQueue, final VirtualThreadProperties virtualThreadProperties, final CollectorRuntimeStatusRegistry runtimeStatusRegistry, - final CollectorRuntimeConfigService runtimeConfigService) { + final CollectorRuntimeConfigService runtimeConfigService, + @Nullable final CollectorLifecycleMaintenanceParticipant collectorLifecycleMaintenance) { this.collectorJobScheduler = collectorJobScheduler; this.collectorJobScheduler.setManageServer(this); this.collectorAlertHandler = collectorAlertHandler; this.commonDataQueue = commonDataQueue; this.runtimeStatusRegistry = runtimeStatusRegistry; this.runtimeConfigService = runtimeConfigService; + this.collectorLifecycleMaintenance = collectorLifecycleMaintenance; this.schedulerProperties = schedulerProperties; this.threadPool = threadPool; this.virtualThreadProperties = virtualThreadProperties == null @@ -219,7 +226,28 @@ public class ManageServer { preChannel.close(); } this.clientChannelTable.put(identity, channel); - this.collectorAlertHandler.online(identity); + } + + public void collectorOnline(String identity, CollectorInfo collectorInfo, boolean submitAlert) { + if (collectorLifecycleMaintenance != null) { + collectorLifecycleMaintenance.collectorOnline(identity, collectorInfo, submitAlert); + } else { + if (submitAlert) { + collectorAlertHandler.online(identity); + } + collectorJobScheduler.collectorGoOnline(identity, collectorInfo); + } + } + + public void collectorOffline(String identity, boolean submitAlert) { + if (collectorLifecycleMaintenance != null) { + collectorLifecycleMaintenance.collectorOffline(identity, submitAlert); + } else { + collectorJobScheduler.collectorGoOffline(identity); + if (submitAlert) { + collectorAlertHandler.offline(identity); + } + } } public void closeChannel(final String identity) { @@ -230,10 +258,10 @@ public class ManageServer { this.runtimeStatusRegistry.remove(identity); Channel channel = this.getChannel(identity); if (channel != null) { - this.collectorJobScheduler.collectorGoOffline(identity); ClusterMsg.Message message = ClusterMsg.Message.newBuilder().setType(ClusterMsg.MessageType.GO_CLOSE).build(); currentServer.sendMsg(channel, message); this.clientChannelTable.remove(identity); + collectorOffline(identity, false); log.info("close collect client success, identity: {}", identity); } } @@ -300,7 +328,7 @@ public class ManageServer { if (identity != null) { ManageServer.this.clientChannelTable.remove(identity); ManageServer.this.runtimeStatusRegistry.remove(identity); - ManageServer.this.collectorJobScheduler.collectorGoOffline(identity); + ManageServer.this.collectorOffline(identity, false); channel.close(); log.info("handle idle event triggered. the client {} is going offline.", identity); } @@ -380,8 +408,7 @@ public class ManageServer { channel.closeFuture(); this.clientChannelTable.remove(collector); this.runtimeStatusRegistry.remove(collector); - this.collectorJobScheduler.collectorGoOffline(collector); - this.collectorAlertHandler.offline(collector); + this.collectorOffline(collector, true); } }); } catch (Exception e) { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/process/CollectorOfflineProcessor.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/process/CollectorOfflineProcessor.java index bccab2c651..38b89c4023 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/process/CollectorOfflineProcessor.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/process/CollectorOfflineProcessor.java @@ -40,7 +40,7 @@ public class CollectorOfflineProcessor implements NettyRemotingProcessor { String collector = message.getIdentity(); log.info("the collector {} actively requests to go offline.", collector); this.manageServer.getRuntimeStatusRegistry().remove(collector); - this.manageServer.getCollectorAndJobScheduler().collectorGoOffline(collector); + this.manageServer.collectorOffline(collector, false); return null; } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/process/CollectorOnlineProcessor.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/process/CollectorOnlineProcessor.java index 9a7e4f4e08..df5d58dc39 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/process/CollectorOnlineProcessor.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/process/CollectorOnlineProcessor.java @@ -54,7 +54,7 @@ public class CollectorOnlineProcessor implements NettyRemotingProcessor { collectorInfo.setIp(clientIP); } this.manageServer.addChannel(collector, ctx.channel()); - this.manageServer.getCollectorAndJobScheduler().collectorGoOnline(collector, collectorInfo); + this.manageServer.collectorOnline(collector, collectorInfo, true); ServerInfo serverInfo = ServerInfo.builder().aesSecret(AesUtil.getDefaultSecretKey()).build(); return ClusterMsg.Message.newBuilder() .setIdentity(message.getIdentity()) diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/process/HeartbeatProcessor.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/process/HeartbeatProcessor.java index b642cd18a0..1607067c97 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/process/HeartbeatProcessor.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/process/HeartbeatProcessor.java @@ -51,7 +51,7 @@ public class HeartbeatProcessor implements NettyRemotingProcessor { log.info("the collector {} is not online.", identity); return null; } else { - this.manageServer.getCollectorAndJobScheduler().collectorGoOnline(identity, null); + this.manageServer.collectorOnline(identity, null, true); } } if (log.isDebugEnabled()) { diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/AlertMetadataMaintenanceParticipantTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/AlertMetadataMaintenanceParticipantTest.java new file mode 100644 index 0000000000..7cfdbef544 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/AlertMetadataMaintenanceParticipantTest.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.doThrow; + +import java.time.Duration; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.apache.hertzbeat.alert.calculate.periodic.PeriodicAlertRuleScheduler; +import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce; +import org.apache.hertzbeat.alert.reduce.AlarmGroupReduce; +import org.junit.jupiter.api.Test; +import org.mockito.InOrder; +import org.mockito.Mockito; + +class AlertMetadataMaintenanceParticipantTest { + + @Test + void eachProducerDrainsBeforeTheNextCutCloses() throws Exception { + PeriodicAlertRuleScheduler periodic = Mockito.mock(PeriodicAlertRuleScheduler.class); + AlarmCommonReduce common = Mockito.mock(AlarmCommonReduce.class); + AlarmGroupReduce group = Mockito.mock(AlarmGroupReduce.class); + CountDownLatch periodicDrainEntered = new CountDownLatch(1); + CountDownLatch releasePeriodic = new CountDownLatch(1); + CountDownLatch commonDrainEntered = new CountDownLatch(1); + CountDownLatch releaseCommon = new CountDownLatch(1); + Mockito.doAnswer(invocation -> { + periodicDrainEntered.countDown(); + releasePeriodic.await(); + return null; + }).when(periodic).awaitDrained(anyLong()); + Mockito.doAnswer(invocation -> { + commonDrainEntered.countDown(); + releaseCommon.await(); + return null; + }).when(common).awaitDrained(anyLong()); + AlertMetadataMaintenanceParticipant participant = + new AlertMetadataMaintenanceParticipant(periodic, common, group); + Thread quiesce = Thread.ofPlatform().start(() -> participant.quiesce(Duration.ofSeconds(30))); + + org.assertj.core.api.Assertions.assertThat(periodicDrainEntered.await(1, TimeUnit.SECONDS)).isTrue(); + verify(common, never()).pauseAdmission(); + releasePeriodic.countDown(); + org.assertj.core.api.Assertions.assertThat(commonDrainEntered.await(1, TimeUnit.SECONDS)).isTrue(); + verify(group, never()).pauseAdmission(); + releaseCommon.countDown(); + quiesce.join(1_000); + + org.assertj.core.api.Assertions.assertThat(quiesce.isAlive()).isFalse(); + } + + @Test + void pausesAndDrainsForwardThenResumesInReverse() throws Exception { + PeriodicAlertRuleScheduler periodic = Mockito.mock(PeriodicAlertRuleScheduler.class); + AlarmCommonReduce common = Mockito.mock(AlarmCommonReduce.class); + AlarmGroupReduce group = Mockito.mock(AlarmGroupReduce.class); + AlertMetadataMaintenanceParticipant participant = + new AlertMetadataMaintenanceParticipant(periodic, common, group); + + participant.quiesce(Duration.ofSeconds(1)); + participant.resume(); + + InOrder order = inOrder(periodic, common, group); + order.verify(periodic).pauseAdmission(); + order.verify(periodic).awaitDrained(anyLong()); + order.verify(common).pauseAdmission(); + order.verify(common).awaitDrained(anyLong()); + order.verify(group).pauseAdmission(); + order.verify(group).awaitDrained(anyLong()); + order.verify(group).resumeAdmission(); + order.verify(common).resumeAdmission(); + order.verify(periodic).resumeAdmission(); + } + + @Test + void drainTimeoutRollsBackInReverseWithSafeFailure() throws Exception { + PeriodicAlertRuleScheduler periodic = Mockito.mock(PeriodicAlertRuleScheduler.class); + AlarmCommonReduce common = Mockito.mock(AlarmCommonReduce.class); + AlarmGroupReduce group = Mockito.mock(AlarmGroupReduce.class); + Mockito.doThrow(new TimeoutException("private-rule")).when(common).awaitDrained(anyLong()); + AlertMetadataMaintenanceParticipant participant = + new AlertMetadataMaintenanceParticipant(periodic, common, group); + + assertThatThrownBy(() -> participant.quiesce(Duration.ofSeconds(1))) + .isInstanceOfSatisfying(MetadataMaintenanceException.class, exception -> { + org.assertj.core.api.Assertions.assertThat(exception.code()) + .isEqualTo(MetadataMaintenanceErrorCode.QUIESCE_TIMEOUT); + org.assertj.core.api.Assertions.assertThat(exception.safeMessage()).doesNotContain("private-rule"); + org.assertj.core.api.Assertions.assertThat(exception.getCause()).isNull(); + }); + + InOrder order = inOrder(periodic, common, group); + order.verify(periodic).pauseAdmission(); + order.verify(periodic).awaitDrained(anyLong()); + order.verify(common).pauseAdmission(); + order.verify(common).awaitDrained(anyLong()); + order.verify(common).resumeAdmission(); + order.verify(periodic).resumeAdmission(); + } + + @Test + void timeoutIsNotHiddenWhenCommonReplayIsRejected() throws Exception { + PeriodicAlertRuleScheduler periodic = Mockito.mock(PeriodicAlertRuleScheduler.class); + AlarmCommonReduce common = Mockito.mock(AlarmCommonReduce.class); + AlarmGroupReduce group = Mockito.mock(AlarmGroupReduce.class); + Mockito.doThrow(new TimeoutException()).when(common).awaitDrained(anyLong()); + doThrow(new IllegalStateException("rejected")).when(common).resumeAdmission(); + AlertMetadataMaintenanceParticipant participant = + new AlertMetadataMaintenanceParticipant(periodic, common, group); + + assertThatThrownBy(() -> participant.quiesce(Duration.ofSeconds(1))) + .isInstanceOfSatisfying(MetadataMaintenanceException.class, exception -> + org.assertj.core.api.Assertions.assertThat(exception.code()) + .isEqualTo(MetadataMaintenanceErrorCode.QUIESCE_TIMEOUT)); + + verify(periodic).resumeAdmission(); + } + + @Test + void interruptIsRestoredWhenCommonReplayIsRejected() throws Exception { + PeriodicAlertRuleScheduler periodic = Mockito.mock(PeriodicAlertRuleScheduler.class); + AlarmCommonReduce common = Mockito.mock(AlarmCommonReduce.class); + AlarmGroupReduce group = Mockito.mock(AlarmGroupReduce.class); + CountDownLatch entered = new CountDownLatch(1); + Mockito.doAnswer(invocation -> { + entered.countDown(); + new CountDownLatch(1).await(); + return null; + }).when(common).awaitDrained(anyLong()); + doThrow(new IllegalStateException("rejected")).when(common).resumeAdmission(); + AlertMetadataMaintenanceParticipant participant = + new AlertMetadataMaintenanceParticipant(periodic, common, group); + java.util.concurrent.atomic.AtomicReference failure = + new java.util.concurrent.atomic.AtomicReference<>(); + java.util.concurrent.atomic.AtomicBoolean interrupted = new java.util.concurrent.atomic.AtomicBoolean(); + Thread thread = Thread.ofPlatform().start(() -> { + try { + participant.quiesce(Duration.ofSeconds(30)); + } catch (MetadataMaintenanceException exception) { + failure.set(exception); + interrupted.set(Thread.currentThread().isInterrupted()); + } + }); + + org.assertj.core.api.Assertions.assertThat(entered.await(1, TimeUnit.SECONDS)).isTrue(); + thread.interrupt(); + thread.join(1_000); + + org.assertj.core.api.Assertions.assertThat(failure.get().code()) + .isEqualTo(MetadataMaintenanceErrorCode.QUIESCE_INTERRUPTED); + org.assertj.core.api.Assertions.assertThat(interrupted).isTrue(); + verify(periodic).resumeAdmission(); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/CollectorLifecycleMaintenanceParticipantTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/CollectorLifecycleMaintenanceParticipantTest.java new file mode 100644 index 0000000000..98ec89cac8 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/CollectorLifecycleMaintenanceParticipantTest.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.alert.calculate.CollectorAlertHandler; +import org.apache.hertzbeat.common.entity.dto.CollectorInfo; +import org.apache.hertzbeat.manager.scheduler.CollectorJobScheduler; +import org.junit.jupiter.api.Test; +import org.mockito.InOrder; +import org.mockito.Mockito; + +class CollectorLifecycleMaintenanceParticipantTest { + + @Test + void runningTransitionsAreSerializedPerIdentityAndLatestRunsLast() throws Exception { + CollectorJobScheduler scheduler = Mockito.mock(CollectorJobScheduler.class); + CollectorAlertHandler alerts = Mockito.mock(CollectorAlertHandler.class); + CountDownLatch onlineAlertEntered = new CountDownLatch(1); + CountDownLatch releaseOnlineAlert = new CountDownLatch(1); + Mockito.doAnswer(invocation -> { + onlineAlertEntered.countDown(); + releaseOnlineAlert.await(); + return null; + }).when(alerts).online("collector-a"); + CollectorLifecycleMaintenanceParticipant participant = + new CollectorLifecycleMaintenanceParticipant(scheduler, alerts); + CollectorInfo info = CollectorInfo.builder().version("current").build(); + Thread online = Thread.ofPlatform().start(() -> participant.collectorOnline("collector-a", info, true)); + assertThat(onlineAlertEntered.await(1, TimeUnit.SECONDS)).isTrue(); + CountDownLatch offlineInvoked = new CountDownLatch(1); + Thread offline = Thread.ofPlatform().start(() -> { + offlineInvoked.countDown(); + participant.collectorOffline("collector-a", false); + }); + assertThat(offlineInvoked.await(1, TimeUnit.SECONDS)).isTrue(); + + releaseOnlineAlert.countDown(); + online.join(1_000); + offline.join(1_000); + + InOrder order = inOrder(alerts, scheduler); + order.verify(alerts).online("collector-a"); + order.verify(scheduler).collectorGoOnline("collector-a", info); + order.verify(scheduler).collectorGoOffline("collector-a"); + verify(alerts, never()).offline("collector-a"); + } + + @Test + void inFlightSpansStatusJobsAndPairedHealthAlertSubmission() throws Exception { + CollectorJobScheduler scheduler = Mockito.mock(CollectorJobScheduler.class); + CollectorAlertHandler alerts = Mockito.mock(CollectorAlertHandler.class); + CountDownLatch schedulerEntered = new CountDownLatch(1); + CountDownLatch releaseScheduler = new CountDownLatch(1); + CountDownLatch alertEntered = new CountDownLatch(1); + CountDownLatch releaseAlert = new CountDownLatch(1); + Mockito.doAnswer(invocation -> { + schedulerEntered.countDown(); + releaseScheduler.await(); + return null; + }).when(scheduler).collectorGoOffline("collector-a"); + Mockito.doAnswer(invocation -> { + alertEntered.countDown(); + releaseAlert.await(); + return null; + }).when(alerts).offline("collector-a"); + CollectorLifecycleMaintenanceParticipant participant = + new CollectorLifecycleMaintenanceParticipant(scheduler, alerts); + Thread transition = Thread.ofPlatform().start(() -> participant.collectorOffline("collector-a", true)); + assertThat(schedulerEntered.await(1, TimeUnit.SECONDS)).isTrue(); + + assertTimeout(participant); + releaseScheduler.countDown(); + assertThat(alertEntered.await(1, TimeUnit.SECONDS)).isTrue(); + assertTimeout(participant); + releaseAlert.countDown(); + transition.join(1_000); + assertThat(transition.isAlive()).isFalse(); + } + + @Test + void pausedTransitionsCoalesceToLatestIntentWithoutChangingAlertSemantics() { + CollectorJobScheduler scheduler = Mockito.mock(CollectorJobScheduler.class); + CollectorAlertHandler alerts = Mockito.mock(CollectorAlertHandler.class); + CollectorLifecycleMaintenanceParticipant participant = + new CollectorLifecycleMaintenanceParticipant(scheduler, alerts); + CollectorInfo first = CollectorInfo.builder().version("first").build(); + CollectorInfo latest = CollectorInfo.builder().version("latest").build(); + participant.quiesce(Duration.ofSeconds(1)); + + participant.collectorOnline("collector-a", first, true); + participant.collectorOffline("collector-a", true); + participant.collectorOnline("collector-a", latest, true); + participant.collectorOffline("collector-b", false); + verify(scheduler, never()).collectorGoOnline(Mockito.anyString(), Mockito.any()); + verify(scheduler, never()).collectorGoOffline(Mockito.anyString()); + + participant.resume(); + + InOrder onlineOrder = inOrder(alerts, scheduler); + onlineOrder.verify(alerts).online("collector-a"); + onlineOrder.verify(scheduler).collectorGoOnline("collector-a", latest); + verify(scheduler).collectorGoOffline("collector-b"); + verify(alerts, never()).offline("collector-b"); + verify(alerts, never()).offline("collector-a"); + } + + @Test + void timeoutReopensAdmissionWithoutReplayingAcrossRunningTransition() throws Exception { + CollectorJobScheduler scheduler = Mockito.mock(CollectorJobScheduler.class); + CollectorAlertHandler alerts = Mockito.mock(CollectorAlertHandler.class); + CountDownLatch firstEntered = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondEntered = new CountDownLatch(1); + Mockito.doAnswer(invocation -> { + firstEntered.countDown(); + releaseFirst.await(); + return null; + }).when(scheduler).collectorGoOffline("collector-a"); + Mockito.doAnswer(invocation -> { + secondEntered.countDown(); + return null; + }).when(scheduler).collectorGoOnline(Mockito.eq("collector-a"), Mockito.any()); + CollectorLifecycleMaintenanceParticipant participant = + new CollectorLifecycleMaintenanceParticipant(scheduler, alerts); + Thread first = Thread.ofPlatform().start(() -> participant.collectorOffline("collector-a", false)); + assertThat(firstEntered.await(1, TimeUnit.SECONDS)).isTrue(); + + assertTimeout(participant); + Thread second = Thread.ofPlatform().start(() -> participant.collectorOnline( + "collector-a", CollectorInfo.builder().version("latest").build(), false)); + second.join(1_000); + assertThat(second.isAlive()).isFalse(); + assertThat(secondEntered.getCount()).isEqualTo(1); + + releaseFirst.countDown(); + assertThat(secondEntered.await(1, TimeUnit.SECONDS)).isTrue(); + first.join(1_000); + } + + @Test + void newerIntentRunsAfterFailedOlderIntentAndOlderIsNotRetried() throws Exception { + CollectorJobScheduler scheduler = Mockito.mock(CollectorJobScheduler.class); + CollectorAlertHandler alerts = Mockito.mock(CollectorAlertHandler.class); + CountDownLatch firstEntered = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondEntered = new CountDownLatch(1); + AtomicReference failure = new AtomicReference<>(); + CollectorInfo info = CollectorInfo.builder().version("old").build(); + Mockito.doAnswer(invocation -> { + firstEntered.countDown(); + releaseFirst.await(); + throw new IllegalStateException("safe fixture failure"); + }).when(scheduler).collectorGoOnline("collector-a", info); + Mockito.doAnswer(invocation -> { + secondEntered.countDown(); + return null; + }).when(scheduler).collectorGoOffline("collector-a"); + CollectorLifecycleMaintenanceParticipant participant = + new CollectorLifecycleMaintenanceParticipant(scheduler, alerts); + Thread first = Thread.ofPlatform().start(() -> { + try { + participant.collectorOnline("collector-a", info, false); + } catch (RuntimeException exception) { + failure.set(exception); + } + }); + assertThat(firstEntered.await(1, TimeUnit.SECONDS)).isTrue(); + participant.collectorOffline("collector-a", false); + + releaseFirst.countDown(); + assertThat(secondEntered.await(1, TimeUnit.SECONDS)).isTrue(); + first.join(1_000); + participant.quiesce(Duration.ofSeconds(1)); + participant.resume(); + + assertThat(failure.get()).isInstanceOf(IllegalStateException.class); + InOrder order = inOrder(scheduler); + order.verify(scheduler).collectorGoOnline("collector-a", info); + order.verify(scheduler).collectorGoOffline("collector-a"); + verify(scheduler, Mockito.times(1)).collectorGoOnline("collector-a", info); + } + + private void assertTimeout(CollectorLifecycleMaintenanceParticipant participant) { + assertThatThrownBy(() -> participant.quiesce(Duration.ZERO)) + .isInstanceOfSatisfying(MetadataMaintenanceException.class, exception -> + assertThat(exception.code()).isEqualTo(MetadataMaintenanceErrorCode.QUIESCE_TIMEOUT)); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceCoordinatorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceCoordinatorTest.java index 525eb2cb2c..bcf2c54c0a 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceCoordinatorTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceCoordinatorTest.java @@ -163,7 +163,8 @@ class MetadataMaintenanceCoordinatorTest { assertThat(exception.getCause()).isNull(); }); - assertThat(events).containsExactly("pause-first", "pause-failing", "resume-first"); + assertThat(events).containsExactly( + "pause-first", "pause-failing", "resume-failing", "resume-first"); assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataMaintenancePhase.RUNNING); } @@ -183,7 +184,7 @@ class MetadataMaintenanceCoordinatorTest { @Override public void resume() { - events.add("unexpected-resume"); + events.add("resume-timeout"); } }; MetadataMaintenanceCoordinator coordinator = new MetadataMaintenanceCoordinator(List.of(timeout)); @@ -193,10 +194,37 @@ class MetadataMaintenanceCoordinatorTest { assertThat(exception.code()).isEqualTo(MetadataMaintenanceErrorCode.QUIESCE_TIMEOUT); assertThat(exception.getCause()).isNull(); }); - assertThat(events).isEmpty(); + assertThat(events).containsExactly("resume-timeout"); assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataMaintenancePhase.RUNNING); } + @Test + void failedParticipantRecoveryCannotBeReportedAsRunning() { + MetadataMaintenanceParticipant participant = new MetadataMaintenanceParticipant() { + @Override + public String participantId() { + return "recovery-failure"; + } + + @Override + public void quiesce(Duration ignored) { + throw MetadataMaintenanceException.quiesceTimeout(); + } + + @Override + public void resume() { + throw MetadataMaintenanceException.resumeFailure(); + } + }; + MetadataMaintenanceCoordinator coordinator = new MetadataMaintenanceCoordinator(List.of(participant)); + + assertThatThrownBy(() -> coordinator.quiesce("operation-a", Duration.ZERO)) + .isInstanceOfSatisfying(MetadataMaintenanceException.class, exception -> + assertThat(exception.code()).isEqualTo(MetadataMaintenanceErrorCode.QUIESCE_TIMEOUT)); + + assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataMaintenancePhase.RECOVERY_REQUIRED); + } + @Test void interruptedQuiesceRestoresInterruptAndRollsBack() throws Exception { CountDownLatch entered = new CountDownLatch(1); diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/MetadataWriteAdmissionStartupContextTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/MetadataWriteAdmissionStartupContextTest.java index 7c8f82d658..39d4876a75 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/MetadataWriteAdmissionStartupContextTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/MetadataWriteAdmissionStartupContextTest.java @@ -19,6 +19,8 @@ import org.apache.hertzbeat.manager.component.status.CalculateStatus; import org.apache.hertzbeat.manager.dao.MonitorDao; import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceCoordinator; import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceParticipant; +import org.apache.hertzbeat.manager.maintenance.AlertMetadataMaintenanceParticipant; +import org.apache.hertzbeat.manager.maintenance.CollectorLifecycleMaintenanceParticipant; import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; import org.apache.hertzbeat.warehouse.store.DataStorageDispatch; import org.apache.hertzbeat.warehouse.store.metadata.JdbcMonitorStatusMetadataWriter; @@ -55,7 +57,9 @@ class MetadataWriteAdmissionStartupContextTest { assertThat(participants) .containsExactly( context.getBean(ServiceDiscoveryWorker.class), - context.getBean(CalculateStatus.class)); + context.getBean(CalculateStatus.class), + context.getBean(CollectorLifecycleMaintenanceParticipant.class), + context.getBean(AlertMetadataMaintenanceParticipant.class)); assertThat(participants).noneMatch(DataStorageDispatch.class::isInstance); assertThat(context.getBeansOfType(MetadataWriteAdmissionCoordinator.class)).hasSize(1); diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupRuntimeBoundaryContextTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupRuntimeBoundaryContextTest.java index 50dd1be331..d5df6cc69c 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupRuntimeBoundaryContextTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupRuntimeBoundaryContextTest.java @@ -166,6 +166,7 @@ class StartupRuntimeBoundaryContextTest { assertFalse(gate.isOpen()); assertTrue(context.containsBeanDefinition("periodicAlertRuleScheduler")); assertTrue(context.containsBeanDefinition("manageServer")); + assertFalse(context.containsBeanDefinition("collectorLifecycleMaintenanceParticipant")); assertTrue(context.containsBeanDefinition("otlpGrpcMetricsService")); assertTrue(context.containsBeanDefinition("alarmGroupReduce")); assertTrue(context.containsBeanDefinition("alarmInhibitReduce")); From 6bc71db9f1cc0756d0722f332b2da40962e560ce Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 03:07:08 +0800 Subject: [PATCH 38/71] Coordinate metadata migration maintenance --- .../CompositeMigrationMaintenanceLease.java | 68 ++++ .../DeadlineConnectionAcquirer.java | 152 ++++++++ ...faultMigrationMaintenanceOrchestrator.java | 246 ++++++++++++ .../DeploymentSingletonAuthority.java | 16 + .../maintenance/DeploymentSingletonLease.java | 15 + .../EmbeddedH2SourceClassifier.java | 98 +++++ .../maintenance/EmbeddedH2SourceGuard.java | 101 +++++ .../MetadataMaintenanceCoordinator.java | 17 + .../MigrationGuardConfiguration.java | 33 ++ .../MigrationMaintenanceErrorCode.java | 21 ++ .../MigrationMaintenanceException.java | 77 ++++ .../MigrationMaintenanceLease.java | 15 + .../MigrationMaintenanceOrchestrator.java | 16 + .../maintenance/MigrationSourceGuard.java | 16 + .../maintenance/MigrationSourceLease.java | 15 + ...tMigrationMaintenanceOrchestratorTest.java | 355 ++++++++++++++++++ .../EmbeddedH2SourceClassifierTest.java | 48 +++ .../EmbeddedH2SourceGuardTest.java | 232 ++++++++++++ .../MetadataMaintenanceCoordinatorTest.java | 14 +- ...adataWriteAdmissionStartupContextTest.java | 19 + .../StartupRuntimeBoundaryContextTest.java | 13 + 21 files changed, 1585 insertions(+), 2 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/CompositeMigrationMaintenanceLease.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/DeadlineConnectionAcquirer.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/DefaultMigrationMaintenanceOrchestrator.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/DeploymentSingletonAuthority.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/DeploymentSingletonLease.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/EmbeddedH2SourceClassifier.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/EmbeddedH2SourceGuard.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationGuardConfiguration.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationMaintenanceErrorCode.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationMaintenanceException.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationMaintenanceLease.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationMaintenanceOrchestrator.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationSourceGuard.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationSourceLease.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/DefaultMigrationMaintenanceOrchestratorTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/EmbeddedH2SourceClassifierTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/EmbeddedH2SourceGuardTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/CompositeMigrationMaintenanceLease.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/CompositeMigrationMaintenanceLease.java new file mode 100644 index 0000000000..b89ae13958 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/CompositeMigrationMaintenanceLease.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import org.apache.hertzbeat.common.transaction.MetadataWriteMaintenanceLease; + +/** Releases one acquired maintenance window in strict reverse order with retryable progress. */ +final class CompositeMigrationMaintenanceLease implements MigrationMaintenanceLease { + + private final DeploymentSingletonLease authorityLease; + private final MigrationSourceLease sourceLease; + private final MetadataMaintenanceLease producerLease; + private final MetadataWriteMaintenanceLease writeLease; + private final Runnable reservationRelease; + private boolean writeReleased; + private boolean producerReleased; + private boolean sourceReleased; + private boolean authorityReleased; + + CompositeMigrationMaintenanceLease( + DeploymentSingletonLease authorityLease, + MigrationSourceLease sourceLease, + MetadataMaintenanceLease producerLease, + MetadataWriteMaintenanceLease writeLease, + Runnable reservationRelease) { + this.authorityLease = authorityLease; + this.sourceLease = sourceLease; + this.producerLease = producerLease; + this.writeLease = writeLease; + this.reservationRelease = reservationRelease; + } + + @Override + public synchronized void close() { + try { + releaseInOrder(); + } catch (MigrationMaintenanceException exception) { + throw exception; + } catch (RuntimeException exception) { + throw MigrationMaintenanceException.resumeFailure(); + } + reservationRelease.run(); + } + + private void releaseInOrder() { + if (!writeReleased) { + writeLease.close(); + writeReleased = true; + } + if (!producerReleased) { + producerLease.resume(); + producerReleased = true; + } + if (!sourceReleased) { + sourceLease.close(); + sourceReleased = true; + } + if (!authorityReleased) { + authorityLease.close(); + authorityReleased = true; + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/DeadlineConnectionAcquirer.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/DeadlineConnectionAcquirer.java new file mode 100644 index 0000000000..324ee97b22 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/DeadlineConnectionAcquirer.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import java.sql.Connection; +import java.sql.SQLException; +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import javax.sql.DataSource; + +/** Bounded JDBC connection acquisition that closes every result arriving after its deadline. */ +final class DeadlineConnectionAcquirer implements AutoCloseable { + + private final DataSource dataSource; + private final ThreadPoolExecutor executor; + + DeadlineConnectionAcquirer(DataSource dataSource) { + this.dataSource = dataSource; + executor = new ThreadPoolExecutor(0, 1, 30, TimeUnit.SECONDS, new SynchronousQueue<>(), + Thread.ofPlatform().daemon(true).name("migration-source-connection", 0).factory()); + } + + Connection acquire(Duration timeout) { + long timeoutNanos; + try { + timeoutNanos = timeout.toNanos(); + } catch (ArithmeticException exception) { + throw MigrationMaintenanceException.invalidRequest(); + } + Attempt attempt = new Attempt(); + try { + executor.execute(() -> connect(attempt)); + } catch (RejectedExecutionException exception) { + if (executor.isShutdown()) { + throw MigrationMaintenanceException.sourceUnavailable(); + } + throw MigrationMaintenanceException.timeout(); + } + return attempt.await(timeoutNanos); + } + + private void connect(Attempt attempt) { + Connection acquired = null; + Throwable failure = null; + try { + acquired = dataSource.getConnection(); + } catch (Throwable connectionFailure) { + failure = connectionFailure; + } + attempt.complete(acquired, failure); + } + + @Override + public void close() { + executor.shutdownNow(); + } + + private static final class Attempt { + + private final Object lock = new Object(); + private final CountDownLatch completed = new CountDownLatch(1); + private Connection connection; + private Throwable failure; + private boolean abandoned; + private boolean finished; + + private Connection await(long timeoutNanos) { + try { + if (completed.await(timeoutNanos, TimeUnit.NANOSECONDS)) { + return claimCompleted(); + } + return abandonOrClaim(); + } catch (InterruptedException exception) { + abandon(); + Thread.currentThread().interrupt(); + throw MigrationMaintenanceException.interrupted(); + } + } + + private void complete(Connection acquired, Throwable acquiredFailure) { + synchronized (lock) { + if (abandoned) { + closeLate(acquired); + } else { + connection = acquired; + failure = acquiredFailure; + } + finished = true; + } + completed.countDown(); + } + + private Connection abandonOrClaim() { + synchronized (lock) { + if (finished) { + return claimCompletedLocked(); + } + abandoned = true; + } + throw MigrationMaintenanceException.timeout(); + } + + private void abandon() { + synchronized (lock) { + if (!finished) { + abandoned = true; + } else { + closeLate(connection); + connection = null; + } + } + } + + private Connection claimCompleted() { + synchronized (lock) { + return claimCompletedLocked(); + } + } + + private Connection claimCompletedLocked() { + if (failure instanceof Error error) { + throw error; + } + if (failure != null || connection == null) { + throw MigrationMaintenanceException.sourceUnavailable(); + } + Connection claimed = connection; + connection = null; + return claimed; + } + + private static void closeLate(Connection lateConnection) { + if (lateConnection == null) { + return; + } + try { + lateConnection.close(); + } catch (SQLException | RuntimeException exception) { + // A late connection never becomes a lease; its details remain private. + } + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/DefaultMigrationMaintenanceOrchestrator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/DefaultMigrationMaintenanceOrchestrator.java new file mode 100644 index 0000000000..3705b9e127 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/DefaultMigrationMaintenanceOrchestrator.java @@ -0,0 +1,246 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import java.time.Duration; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.LongSupplier; +import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionCoordinator; +import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionErrorCode; +import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionException; +import org.apache.hertzbeat.common.transaction.MetadataWriteMaintenanceLease; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** Composes deployment, source, producer, and transaction fences without owning their state machines. */ +@Component +public final class DefaultMigrationMaintenanceOrchestrator implements MigrationMaintenanceOrchestrator { + + private final ReentrantLock lock = new ReentrantLock(); + private final DeploymentSingletonAuthority deploymentAuthority; + private final MigrationSourceGuard sourceGuard; + private final MetadataMaintenanceCoordinator producerCoordinator; + private final MetadataWriteAdmissionCoordinator writeCoordinator; + private final LongSupplier ticker; + private String operationId; + private Object ownerToken; + private MetadataMaintenanceLease recoveryProducerLease; + private boolean recoveryProducerCoordinatorRequired; + private MigrationSourceLease recoverySourceLease; + private DeploymentSingletonLease recoveryAuthorityLease; + + @Autowired + public DefaultMigrationMaintenanceOrchestrator( + DeploymentSingletonAuthority deploymentAuthority, + MigrationSourceGuard sourceGuard, + MetadataMaintenanceCoordinator producerCoordinator, + MetadataWriteAdmissionCoordinator writeCoordinator) { + this(deploymentAuthority, sourceGuard, producerCoordinator, writeCoordinator, System::nanoTime); + } + + DefaultMigrationMaintenanceOrchestrator( + DeploymentSingletonAuthority deploymentAuthority, + MigrationSourceGuard sourceGuard, + MetadataMaintenanceCoordinator producerCoordinator, + MetadataWriteAdmissionCoordinator writeCoordinator, + LongSupplier ticker) { + this.deploymentAuthority = deploymentAuthority; + this.sourceGuard = sourceGuard; + this.producerCoordinator = producerCoordinator; + this.writeCoordinator = writeCoordinator; + this.ticker = ticker; + } + + @Override + public MigrationMaintenanceLease acquire(String requestedOperationId, Duration timeout) { + requireRequest(requestedOperationId); + MaintenanceDeadline deadline; + try { + deadline = MaintenanceDeadline.start(timeout, ticker); + } catch (MetadataMaintenanceException exception) { + throw MigrationMaintenanceException.invalidRequest(); + } + Object token = reserve(requestedOperationId); + DeploymentSingletonLease authorityLease = null; + MigrationSourceLease sourceLease = null; + MetadataMaintenanceLease producerLease = null; + try { + authorityLease = deploymentAuthority.acquire(requestedOperationId, deadline.remaining()); + sourceLease = sourceGuard.fence(requestedOperationId, deadline.remaining()); + producerLease = producerCoordinator.quiesce(requestedOperationId, deadline.remaining()); + MetadataWriteMaintenanceLease writeLease = + writeCoordinator.acquire(requestedOperationId, deadline.remaining()); + return new CompositeMigrationMaintenanceLease( + authorityLease, sourceLease, producerLease, writeLease, () -> releaseReservation(token)); + } catch (Error error) { + if (cleanupFailedAcquisition(error, producerLease, sourceLease, authorityLease)) { + releaseReservation(token); + } else { + retainRecovery(producerLease, sourceLease, authorityLease); + } + throw error; + } catch (RuntimeException exception) { + MigrationMaintenanceException primary = mapFailure(exception); + if (cleanupFailedAcquisition(primary, producerLease, sourceLease, authorityLease)) { + releaseReservation(token); + } else { + retainRecovery(producerLease, sourceLease, authorityLease); + } + throw primary; + } + } + + private Object reserve(String requestedOperationId) { + lock.lock(); + try { + if (ownerToken != null) { + if (!requestedOperationId.equals(operationId) + || !hasRecovery()) { + throw MigrationMaintenanceException.operationConflict(); + } + recoverFailedAcquisition(); + } + Object token = new Object(); + ownerToken = token; + operationId = requestedOperationId; + return token; + } finally { + lock.unlock(); + } + } + + private boolean cleanupFailedAcquisition( + Throwable primary, + MetadataMaintenanceLease producerLease, + MigrationSourceLease sourceLease, + DeploymentSingletonLease authorityLease) { + boolean interrupted = Thread.currentThread().isInterrupted(); + boolean producerReleased = producerCoordinator.snapshot().phase() == MetadataMaintenancePhase.RUNNING; + if (producerLease != null) { + producerReleased = suppressCleanup(primary, producerLease::resume); + } + boolean sourceReleased = sourceLease == null; + if (producerReleased && sourceLease != null) { + sourceReleased = suppressCleanup(primary, sourceLease::close); + } + boolean authorityReleased = authorityLease == null; + if (producerReleased && sourceReleased && authorityLease != null) { + authorityReleased = suppressCleanup(primary, authorityLease::close); + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + return producerReleased && sourceReleased && authorityReleased; + } + + private boolean suppressCleanup(Throwable primary, Runnable cleanup) { + try { + cleanup.run(); + return true; + } catch (RuntimeException exception) { + primary.addSuppressed(MigrationMaintenanceException.resumeFailure()); + return false; + } + } + + private void retainRecovery( + MetadataMaintenanceLease producerLease, + MigrationSourceLease sourceLease, + DeploymentSingletonLease authorityLease) { + lock.lock(); + try { + recoveryProducerLease = producerLease; + recoveryProducerCoordinatorRequired = producerLease == null + && producerCoordinator.snapshot().phase() != MetadataMaintenancePhase.RUNNING; + recoverySourceLease = sourceLease; + recoveryAuthorityLease = authorityLease; + } finally { + lock.unlock(); + } + } + + private void recoverFailedAcquisition() { + try { + if (recoveryProducerLease != null) { + recoveryProducerLease.resume(); + recoveryProducerLease = null; + } + if (recoveryProducerCoordinatorRequired) { + producerCoordinator.recover(operationId); + recoveryProducerCoordinatorRequired = false; + } + if (recoverySourceLease != null) { + recoverySourceLease.close(); + recoverySourceLease = null; + } + if (recoveryAuthorityLease != null) { + recoveryAuthorityLease.close(); + recoveryAuthorityLease = null; + } + ownerToken = null; + operationId = null; + } catch (RuntimeException exception) { + throw MigrationMaintenanceException.resumeFailure(); + } + } + + private boolean hasRecovery() { + return recoveryProducerLease != null + || recoveryProducerCoordinatorRequired + || recoverySourceLease != null + || recoveryAuthorityLease != null; + } + + private MigrationMaintenanceException mapFailure(RuntimeException exception) { + if (exception instanceof MigrationMaintenanceException migrationFailure) { + return migrationFailure; + } + if (exception instanceof MetadataMaintenanceException maintenanceFailure) { + return switch (maintenanceFailure.code()) { + case INVALID_REQUEST -> MigrationMaintenanceException.invalidRequest(); + case OPERATION_CONFLICT, STALE_LEASE -> MigrationMaintenanceException.operationConflict(); + case QUIESCE_TIMEOUT -> MigrationMaintenanceException.timeout(); + case QUIESCE_INTERRUPTED -> MigrationMaintenanceException.interrupted(); + case PARTICIPANT_FAILURE, RESUME_FAILURE -> MigrationMaintenanceException.maintenanceFailure(); + }; + } + if (exception instanceof MetadataWriteAdmissionException writeFailure) { + MetadataWriteAdmissionErrorCode code = writeFailure.code(); + return switch (code) { + case INVALID_REQUEST -> MigrationMaintenanceException.invalidRequest(); + case OPERATION_CONFLICT, MAINTENANCE_ACTIVE -> MigrationMaintenanceException.operationConflict(); + case DRAIN_TIMEOUT -> MigrationMaintenanceException.timeout(); + case ACQUISITION_INTERRUPTED -> MigrationMaintenanceException.interrupted(); + }; + } + return MigrationMaintenanceException.maintenanceFailure(); + } + + private void releaseReservation(Object token) { + lock.lock(); + try { + if (ownerToken == token) { + ownerToken = null; + operationId = null; + recoveryProducerLease = null; + recoveryProducerCoordinatorRequired = false; + recoverySourceLease = null; + recoveryAuthorityLease = null; + } + } finally { + lock.unlock(); + } + } + + private void requireRequest(String requestedOperationId) { + if (requestedOperationId == null || requestedOperationId.isBlank()) { + throw MigrationMaintenanceException.invalidRequest(); + } + } + +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/DeploymentSingletonAuthority.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/DeploymentSingletonAuthority.java new file mode 100644 index 0000000000..78eed1e15f --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/DeploymentSingletonAuthority.java @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import java.time.Duration; + +/** Proves and fences single-manager ownership for one logical deployment. */ +public interface DeploymentSingletonAuthority { + + DeploymentSingletonLease acquire(String operationId, Duration timeout); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/DeploymentSingletonLease.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/DeploymentSingletonLease.java new file mode 100644 index 0000000000..7feb82f16f --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/DeploymentSingletonLease.java @@ -0,0 +1,15 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +/** Owner capability for one authoritative deployment-singleton fence. */ +public interface DeploymentSingletonLease extends AutoCloseable { + + @Override + void close(); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/EmbeddedH2SourceClassifier.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/EmbeddedH2SourceClassifier.java new file mode 100644 index 0000000000..490135c840 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/EmbeddedH2SourceClassifier.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import java.nio.file.Path; +import java.util.Locale; + +/** Pure embedded-H2 source-safety classification. */ +final class EmbeddedH2SourceClassifier { + + private static final String H2_PREFIX = "jdbc:h2:"; + + private EmbeddedH2SourceClassifier() { + } + + static boolean isSafeEmbeddedSource(String productName, String jdbcUrl) { + if (productName == null || !"h2".equals(productName.trim().toLowerCase(Locale.ROOT)) + || jdbcUrl == null) { + return false; + } + String url = jdbcUrl.trim().toLowerCase(Locale.ROOT); + if (!url.startsWith(H2_PREFIX) || hasUnsafeSetting(url)) { + return false; + } + String location = url.substring(H2_PREFIX.length()); + if (location.startsWith("mem:")) { + return !isRemote(location.substring("mem:".length())); + } + String fileLocation = location.startsWith("file:") + ? location.substring("file:".length()) + : location; + return !isRemote(fileLocation) && (location.startsWith("file:") + || location.startsWith("./") + || location.startsWith("../") + || location.startsWith(".\\") + || location.startsWith("..\\") + || location.startsWith("~/") + || location.startsWith("/") + || isWindowsDrive(location)); + } + + static boolean matchesConfiguredSource(String configuredUrl, String actualUrl) { + String configuredLocation = sourceLocation(configuredUrl); + String actualLocation = sourceLocation(actualUrl); + if (configuredLocation == null || actualLocation == null) { + return false; + } + if (configuredLocation.startsWith("mem:") || actualLocation.startsWith("mem:")) { + return configuredLocation.equals(actualLocation); + } + try { + return localPath(configuredLocation).equals(localPath(actualLocation)); + } catch (RuntimeException exception) { + return false; + } + } + + private static String sourceLocation(String jdbcUrl) { + if (jdbcUrl == null || !jdbcUrl.regionMatches(true, 0, H2_PREFIX, 0, H2_PREFIX.length())) { + return null; + } + String location = jdbcUrl.substring(H2_PREFIX.length()).split(";", 2)[0]; + return location.regionMatches(true, 0, "file:", 0, "file:".length()) + ? location.substring("file:".length()) : location; + } + + private static Path localPath(String location) { + String expanded = location.startsWith("~/") || location.startsWith("~\\") + ? System.getProperty("user.home") + location.substring(1) : location; + return Path.of(expanded).toAbsolutePath().normalize(); + } + + private static boolean isRemote(String location) { + return location.contains("://") || location.startsWith("//") || location.startsWith("\\\\"); + } + + private static boolean isWindowsDrive(String location) { + return location.length() >= 3 + && Character.isLetter(location.charAt(0)) + && location.charAt(1) == ':' + && (location.charAt(2) == '\\' || location.charAt(2) == '/'); + } + + private static boolean hasUnsafeSetting(String url) { + for (String setting : url.split(";")) { + String normalized = setting.strip(); + if (normalized.startsWith("auto_server") || normalized.equals("file_lock=no")) { + return true; + } + } + return false; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/EmbeddedH2SourceGuard.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/EmbeddedH2SourceGuard.java new file mode 100644 index 0000000000..9fc057a20e --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/EmbeddedH2SourceGuard.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.sql.DataSource; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.boot.jdbc.autoconfigure.DataSourceProperties; +import org.springframework.stereotype.Component; + +/** Holds an embedded H2 connection after proving its configured local access mode is safe. */ +@Component +@ConditionalOnNormalBusinessRuntime +public final class EmbeddedH2SourceGuard implements MigrationSourceGuard, DisposableBean { + + private final DataSourceProperties dataSourceProperties; + private final DeadlineConnectionAcquirer connectionAcquirer; + + public EmbeddedH2SourceGuard(DataSource dataSource, DataSourceProperties dataSourceProperties) { + this.dataSourceProperties = dataSourceProperties; + this.connectionAcquirer = new DeadlineConnectionAcquirer(dataSource); + } + + @Override + public MigrationSourceLease fence(String operationId, Duration timeout) { + requireRequest(operationId, timeout); + String configuredUrl = dataSourceProperties.getUrl(); + if (!EmbeddedH2SourceClassifier.isSafeEmbeddedSource("H2", configuredUrl)) { + throw MigrationMaintenanceException.sourceUnavailable(); + } + Connection connection = connectionAcquirer.acquire(timeout); + try { + DatabaseMetaData metadata = connection.getMetaData(); + String productName = metadata.getDatabaseProductName(); + String actualUrl = metadata.getURL(); + if (!EmbeddedH2SourceClassifier.isSafeEmbeddedSource(productName, actualUrl) + || !EmbeddedH2SourceClassifier.matchesConfiguredSource(configuredUrl, actualUrl)) { + closeRejectedConnection(connection); + throw MigrationMaintenanceException.sourceUnavailable(); + } + return new ConnectionSourceLease(connection); + } catch (MigrationMaintenanceException exception) { + throw exception; + } catch (SQLException | RuntimeException exception) { + closeRejectedConnection(connection); + throw MigrationMaintenanceException.sourceUnavailable(); + } + } + + @Override + public void destroy() { + connectionAcquirer.close(); + } + + private void requireRequest(String operationId, Duration timeout) { + if (operationId == null || operationId.isBlank() || timeout == null || timeout.isNegative()) { + throw MigrationMaintenanceException.invalidRequest(); + } + } + + private void closeRejectedConnection(Connection connection) { + try { + connection.close(); + } catch (SQLException | RuntimeException exception) { + // The stable source failure remains primary. + } + } + + private static final class ConnectionSourceLease implements MigrationSourceLease { + + private final Connection connection; + private final AtomicBoolean closed = new AtomicBoolean(); + + private ConnectionSourceLease(Connection connection) { + this.connection = connection; + } + + @Override + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + try { + connection.close(); + } catch (SQLException | RuntimeException exception) { + closed.set(false); + throw MigrationMaintenanceException.resumeFailure(); + } + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceCoordinator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceCoordinator.java index de8e8f26b6..7755c6bbe9 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceCoordinator.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceCoordinator.java @@ -73,6 +73,23 @@ public final class MetadataMaintenanceCoordinator { } } + void recover(String requestedOperationId) { + requireOperationId(requestedOperationId); + lock.lock(); + try { + if (phase != MetadataMaintenancePhase.RECOVERY_REQUIRED + || !requestedOperationId.equals(operationId)) { + throw MetadataMaintenanceException.operationConflict(); + } + if (!resumeAllParticipants()) { + throw MetadataMaintenanceException.resumeFailure(); + } + reopen(); + } finally { + lock.unlock(); + } + } + void resume(String resumedOperationId, long resumedEpoch, Object resumedToken) { lock.lock(); try { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationGuardConfiguration.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationGuardConfiguration.java new file mode 100644 index 0000000000..d2ba64220a --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationGuardConfiguration.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** Fail-closed fallbacks for maintenance facts that the runtime cannot prove. */ +@Configuration(proxyBeanMethods = false) +public class MigrationGuardConfiguration { + + @Bean + @ConditionalOnMissingBean(DeploymentSingletonAuthority.class) + DeploymentSingletonAuthority unavailableDeploymentSingletonAuthority() { + return (operationId, timeout) -> { + throw MigrationMaintenanceException.deploymentAuthorityUnavailable(); + }; + } + + @Bean + @ConditionalOnMissingBean(MigrationSourceGuard.class) + MigrationSourceGuard unavailableMigrationSourceGuard() { + return (operationId, timeout) -> { + throw MigrationMaintenanceException.sourceUnavailable(); + }; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationMaintenanceErrorCode.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationMaintenanceErrorCode.java new file mode 100644 index 0000000000..1d54e277cc --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationMaintenanceErrorCode.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +/** Stable safe failure categories for migration maintenance acquisition and release. */ +public enum MigrationMaintenanceErrorCode { + MIGRATION_DEPLOYMENT_AUTHORITY_UNAVAILABLE, + MIGRATION_SOURCE_UNAVAILABLE, + MIGRATION_MULTI_NODE_UNSUPPORTED, + MIGRATION_OPERATION_CONFLICT, + MIGRATION_MAINTENANCE_TIMEOUT, + MIGRATION_MAINTENANCE_INTERRUPTED, + MIGRATION_MAINTENANCE_FAILURE, + MIGRATION_RESUME_FAILURE, + INVALID_REQUEST +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationMaintenanceException.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationMaintenanceException.java new file mode 100644 index 0000000000..fb21a4efc6 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationMaintenanceException.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +/** Secret-free migration maintenance failure. */ +public final class MigrationMaintenanceException extends RuntimeException { + + private final MigrationMaintenanceErrorCode code; + + private MigrationMaintenanceException(MigrationMaintenanceErrorCode code, String message) { + super(message); + this.code = code; + } + + public MigrationMaintenanceErrorCode code() { + return code; + } + + public String safeMessage() { + return getMessage(); + } + + public static MigrationMaintenanceException deploymentAuthorityUnavailable() { + return failure(MigrationMaintenanceErrorCode.MIGRATION_DEPLOYMENT_AUTHORITY_UNAVAILABLE, + "Migration deployment authority is unavailable"); + } + + public static MigrationMaintenanceException sourceUnavailable() { + return failure(MigrationMaintenanceErrorCode.MIGRATION_SOURCE_UNAVAILABLE, + "Migration metadata source is unavailable"); + } + + public static MigrationMaintenanceException multiNodeUnsupported() { + return failure(MigrationMaintenanceErrorCode.MIGRATION_MULTI_NODE_UNSUPPORTED, + "Multi-node metadata migration is unsupported"); + } + + static MigrationMaintenanceException operationConflict() { + return failure(MigrationMaintenanceErrorCode.MIGRATION_OPERATION_CONFLICT, + "Migration maintenance operation is already active"); + } + + static MigrationMaintenanceException timeout() { + return failure(MigrationMaintenanceErrorCode.MIGRATION_MAINTENANCE_TIMEOUT, + "Migration maintenance acquisition timed out"); + } + + static MigrationMaintenanceException interrupted() { + return failure(MigrationMaintenanceErrorCode.MIGRATION_MAINTENANCE_INTERRUPTED, + "Migration maintenance acquisition was interrupted"); + } + + static MigrationMaintenanceException maintenanceFailure() { + return failure(MigrationMaintenanceErrorCode.MIGRATION_MAINTENANCE_FAILURE, + "Migration maintenance acquisition failed"); + } + + static MigrationMaintenanceException resumeFailure() { + return failure(MigrationMaintenanceErrorCode.MIGRATION_RESUME_FAILURE, + "Migration maintenance release failed"); + } + + static MigrationMaintenanceException invalidRequest() { + return failure(MigrationMaintenanceErrorCode.INVALID_REQUEST, + "Migration maintenance request is invalid"); + } + + private static MigrationMaintenanceException failure( + MigrationMaintenanceErrorCode code, String message) { + return new MigrationMaintenanceException(code, message); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationMaintenanceLease.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationMaintenanceLease.java new file mode 100644 index 0000000000..841a9544cd --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationMaintenanceLease.java @@ -0,0 +1,15 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +/** Owner capability for one fully acquired migration maintenance window. */ +public interface MigrationMaintenanceLease extends AutoCloseable { + + @Override + void close(); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationMaintenanceOrchestrator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationMaintenanceOrchestrator.java new file mode 100644 index 0000000000..c84579a0fd --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationMaintenanceOrchestrator.java @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import java.time.Duration; + +/** Acquires the complete process-local maintenance window required before metadata migration. */ +public interface MigrationMaintenanceOrchestrator { + + MigrationMaintenanceLease acquire(String operationId, Duration timeout); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationSourceGuard.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationSourceGuard.java new file mode 100644 index 0000000000..cb801e6d6c --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationSourceGuard.java @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import java.time.Duration; + +/** Fences a metadata source whose local access mode is safe for migration. */ +public interface MigrationSourceGuard { + + MigrationSourceLease fence(String operationId, Duration timeout); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationSourceLease.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationSourceLease.java new file mode 100644 index 0000000000..5aefff2025 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationSourceLease.java @@ -0,0 +1,15 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +/** Capability that releases one safe metadata-source fence. */ +public interface MigrationSourceLease extends AutoCloseable { + + @Override + void close(); +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/DefaultMigrationMaintenanceOrchestratorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/DefaultMigrationMaintenanceOrchestratorTest.java new file mode 100644 index 0000000000..ae0112e82a --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/DefaultMigrationMaintenanceOrchestratorTest.java @@ -0,0 +1,355 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionCoordinator; +import org.apache.hertzbeat.common.transaction.MetadataWriteMaintenanceLease; +import org.junit.jupiter.api.Test; +import org.mockito.InOrder; +import org.mockito.Mockito; + +class DefaultMigrationMaintenanceOrchestratorTest { + + @Test + void acquiresForwardWithOneDeadlineAndReleasesInReverse() { + Harness harness = harness(); + AtomicLong ticker = new AtomicLong(10); + DefaultMigrationMaintenanceOrchestrator orchestrator = new DefaultMigrationMaintenanceOrchestrator( + harness.deploymentAuthority, harness.sourceGuard, + harness.producerCoordinator, harness.writeCoordinator, ticker::get); + when(harness.deploymentAuthority.acquire(eq("operation-a"), any())).thenAnswer(invocation -> { + assertThat((Duration) invocation.getArgument(1)).isEqualTo(Duration.ofNanos(100)); + ticker.addAndGet(10); + return harness.authorityLease; + }); + when(harness.sourceGuard.fence(eq("operation-a"), any())).thenAnswer(invocation -> { + assertThat((Duration) invocation.getArgument(1)).isEqualTo(Duration.ofNanos(90)); + ticker.addAndGet(20); + return harness.sourceLease; + }); + when(harness.producerCoordinator.quiesce(eq("operation-a"), any())).thenAnswer(invocation -> { + assertThat((Duration) invocation.getArgument(1)).isEqualTo(Duration.ofNanos(70)); + ticker.addAndGet(30); + return harness.producerLease; + }); + when(harness.writeCoordinator.acquire(eq("operation-a"), any())).thenAnswer(invocation -> { + assertThat((Duration) invocation.getArgument(1)).isEqualTo(Duration.ofNanos(40)); + return harness.writeLease; + }); + + MigrationMaintenanceLease lease = orchestrator.acquire("operation-a", Duration.ofNanos(100)); + lease.close(); + + InOrder order = inOrder(harness.deploymentAuthority, harness.sourceGuard, harness.producerCoordinator, + harness.writeCoordinator, harness.writeLease, harness.producerLease, + harness.sourceLease, harness.authorityLease); + order.verify(harness.deploymentAuthority).acquire(eq("operation-a"), any()); + order.verify(harness.sourceGuard).fence(eq("operation-a"), any()); + order.verify(harness.producerCoordinator).quiesce(eq("operation-a"), any()); + order.verify(harness.writeCoordinator).acquire(eq("operation-a"), any()); + order.verify(harness.writeLease).close(); + order.verify(harness.producerLease).resume(); + order.verify(harness.sourceLease).close(); + order.verify(harness.authorityLease).close(); + } + + @Test + void deploymentAuthorityUnknownOrMultiFailsBeforeSourceOrLocalPause() { + Harness unknown = harness(); + when(unknown.deploymentAuthority.acquire(eq("operation-a"), any())) + .thenThrow(MigrationMaintenanceException.deploymentAuthorityUnavailable()); + DefaultMigrationMaintenanceOrchestrator unknownOrchestrator = unknown.orchestrator(); + assertThatThrownBy(() -> unknownOrchestrator.acquire("operation-a", Duration.ZERO)) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, exception -> + assertThat(exception.code()) + .isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_DEPLOYMENT_AUTHORITY_UNAVAILABLE)); + verify(unknown.sourceGuard, never()).fence(any(), any()); + verify(unknown.producerCoordinator, never()).quiesce(any(), any()); + verify(unknown.writeCoordinator, never()).acquire(any(), any()); + + Harness multi = harness(); + when(multi.deploymentAuthority.acquire(eq("operation-a"), any())) + .thenThrow(MigrationMaintenanceException.multiNodeUnsupported()); + assertThatThrownBy(() -> multi.orchestrator().acquire("operation-a", Duration.ZERO)) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, exception -> + assertThat(exception.code()) + .isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_MULTI_NODE_UNSUPPORTED)); + verify(multi.producerCoordinator, never()).quiesce(any(), any()); + } + + @Test + void secondAndThirdStepFailuresCleanUpInReverseWithoutReplacingPrimary() { + Harness second = harness(); + when(second.sourceGuard.fence(eq("operation-a"), any())).thenReturn(second.sourceLease); + when(second.producerCoordinator.quiesce(eq("operation-a"), any())) + .thenThrow(MetadataMaintenanceException.quiesceTimeout()); + Mockito.doThrow(new IllegalStateException("private-cleanup")).when(second.sourceLease).close(); + assertThatThrownBy(() -> second.orchestrator().acquire("operation-a", Duration.ZERO)) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, exception -> { + assertThat(exception.code()).isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_MAINTENANCE_TIMEOUT); + assertThat(exception.getSuppressed()).hasSize(1); + assertThat(exception.getMessage()).doesNotContain("private"); + }); + + Harness third = harness(); + when(third.sourceGuard.fence(eq("operation-a"), any())).thenReturn(third.sourceLease); + when(third.producerCoordinator.quiesce(eq("operation-a"), any())).thenReturn(third.producerLease); + when(third.writeCoordinator.acquire(eq("operation-a"), any())) + .thenThrow(new IllegalStateException("private-write")); + assertThatThrownBy(() -> third.orchestrator().acquire("operation-a", Duration.ZERO)) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, exception -> + assertThat(exception.code()) + .isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_MAINTENANCE_FAILURE)); + InOrder cleanup = inOrder(third.producerLease, third.sourceLease, third.authorityLease); + cleanup.verify(third.producerLease).resume(); + cleanup.verify(third.sourceLease).close(); + cleanup.verify(third.authorityLease).close(); + } + + @Test + void failedAcquisitionRecoveryRetainsFenceAndOwnershipUntilSameOperationRetries() { + Harness harness = harness(); + when(harness.sourceGuard.fence(eq("operation-a"), any())) + .thenReturn(harness.sourceLease) + .thenThrow(MigrationMaintenanceException.sourceUnavailable()); + when(harness.producerCoordinator.quiesce(eq("operation-a"), any())).thenReturn(harness.producerLease); + when(harness.writeCoordinator.acquire(eq("operation-a"), any())) + .thenThrow(new IllegalStateException("private-write")); + Mockito.doThrow(new IllegalStateException("private-resume")) + .doNothing().when(harness.producerLease).resume(); + DefaultMigrationMaintenanceOrchestrator orchestrator = harness.orchestrator(); + + assertThatThrownBy(() -> orchestrator.acquire("operation-a", Duration.ZERO)) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, exception -> { + assertThat(exception.code()) + .isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_MAINTENANCE_FAILURE); + assertThat(exception.getSuppressed()).hasSize(1); + }); + verify(harness.sourceLease, never()).close(); + verify(harness.authorityLease, never()).close(); + assertThatThrownBy(() -> orchestrator.acquire("operation-b", Duration.ZERO)) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, exception -> + assertThat(exception.code()) + .isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_OPERATION_CONFLICT)); + + assertThatThrownBy(() -> orchestrator.acquire("operation-a", Duration.ZERO)) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, exception -> + assertThat(exception.code()) + .isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_SOURCE_UNAVAILABLE)); + verify(harness.producerLease, Mockito.times(2)).resume(); + verify(harness.sourceLease).close(); + verify(harness.authorityLease, Mockito.times(2)).close(); + } + + @Test + void producerRecoveryWithoutLeaseRetainsFenceUntilSameOperationRecoversCoordinator() { + Harness harness = harness(); + when(harness.sourceGuard.fence(eq("operation-a"), any())) + .thenReturn(harness.sourceLease) + .thenThrow(MigrationMaintenanceException.sourceUnavailable()); + when(harness.producerCoordinator.quiesce(eq("operation-a"), any())) + .thenThrow(MetadataMaintenanceException.participantFailure()); + AtomicReference producerState = new AtomicReference<>( + new MetadataMaintenanceSnapshot(MetadataMaintenancePhase.RECOVERY_REQUIRED, "operation-a", 1)); + when(harness.producerCoordinator.snapshot()).thenAnswer(invocation -> producerState.get()); + Mockito.doAnswer(invocation -> { + producerState.set(new MetadataMaintenanceSnapshot(MetadataMaintenancePhase.RUNNING, null, 1)); + return null; + }).when(harness.producerCoordinator).recover("operation-a"); + DefaultMigrationMaintenanceOrchestrator orchestrator = harness.orchestrator(); + + assertThatThrownBy(() -> orchestrator.acquire("operation-a", Duration.ofSeconds(1))) + .isInstanceOf(MigrationMaintenanceException.class); + verify(harness.sourceLease, never()).close(); + assertThatThrownBy(() -> orchestrator.acquire("operation-b", Duration.ZERO)) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, exception -> + assertThat(exception.code()).isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_OPERATION_CONFLICT)); + + assertThatThrownBy(() -> orchestrator.acquire("operation-a", Duration.ofSeconds(1))) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, exception -> + assertThat(exception.code()) + .isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_SOURCE_UNAVAILABLE)); + verify(harness.producerCoordinator).recover("operation-a"); + verify(harness.sourceLease).close(); + verify(harness.authorityLease, Mockito.times(2)).close(); + } + + @Test + void resumeFailureRetainsTopologyAndSameOwnerCanRetry() { + Harness harness = harness(); + when(harness.sourceGuard.fence(eq("operation-a"), any())).thenReturn(harness.sourceLease); + when(harness.producerCoordinator.quiesce(eq("operation-a"), any())).thenReturn(harness.producerLease); + when(harness.writeCoordinator.acquire(eq("operation-a"), any())).thenReturn(harness.writeLease); + Mockito.doThrow(new IllegalStateException()).doNothing().when(harness.producerLease).resume(); + MigrationMaintenanceLease lease = harness.orchestrator().acquire("operation-a", Duration.ZERO); + + assertThatThrownBy(lease::close) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, exception -> + assertThat(exception.code()).isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_RESUME_FAILURE)); + verify(harness.sourceLease, never()).close(); + verify(harness.authorityLease, never()).close(); + lease.close(); + verify(harness.writeLease).close(); + verify(harness.producerLease, Mockito.times(2)).resume(); + verify(harness.sourceLease).close(); + verify(harness.authorityLease).close(); + } + + @Test + void concurrentAcquisitionHasOnlyOneOwnerCapability() throws Exception { + Harness harness = harness(); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + when(harness.sourceGuard.fence(eq("operation-a"), any())).thenAnswer(invocation -> { + entered.countDown(); + release.await(); + return harness.sourceLease; + }); + when(harness.producerCoordinator.quiesce(eq("operation-a"), any())).thenReturn(harness.producerLease); + when(harness.writeCoordinator.acquire(eq("operation-a"), any())).thenReturn(harness.writeLease); + DefaultMigrationMaintenanceOrchestrator orchestrator = harness.orchestrator(); + AtomicReference owner = new AtomicReference<>(); + Thread thread = Thread.ofPlatform().start(() -> + owner.set(orchestrator.acquire("operation-a", Duration.ofSeconds(30)))); + assertThat(entered.await(1, TimeUnit.SECONDS)).isTrue(); + + assertThatThrownBy(() -> orchestrator.acquire("operation-b", Duration.ZERO)) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, exception -> + assertThat(exception.code()).isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_OPERATION_CONFLICT)); + assertThatThrownBy(() -> orchestrator.acquire("operation-a", Duration.ZERO)) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, exception -> + assertThat(exception.code()).isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_OPERATION_CONFLICT)); + release.countDown(); + thread.join(1_000); + owner.get().close(); + } + + @Test + void sharedAuthorityNotSeparateEmbeddedSourcesDeterminesSingletonOwnership() { + Harness first = harness(); + Harness second = harness(); + FakeSingletonAuthority authority = new FakeSingletonAuthority(); + when(first.sourceGuard.fence(eq("operation-a"), any())).thenReturn(first.sourceLease); + when(first.producerCoordinator.quiesce(eq("operation-a"), any())).thenReturn(first.producerLease); + when(first.writeCoordinator.acquire(eq("operation-a"), any())).thenReturn(first.writeLease); + when(second.sourceGuard.fence(eq("operation-b"), any())).thenReturn(second.sourceLease); + when(second.producerCoordinator.quiesce(eq("operation-b"), any())).thenReturn(second.producerLease); + when(second.writeCoordinator.acquire(eq("operation-b"), any())).thenReturn(second.writeLease); + DefaultMigrationMaintenanceOrchestrator firstOrchestrator = new DefaultMigrationMaintenanceOrchestrator( + authority, first.sourceGuard, first.producerCoordinator, first.writeCoordinator); + DefaultMigrationMaintenanceOrchestrator secondOrchestrator = new DefaultMigrationMaintenanceOrchestrator( + authority, second.sourceGuard, second.producerCoordinator, second.writeCoordinator); + + MigrationMaintenanceLease firstLease = firstOrchestrator.acquire("operation-a", Duration.ZERO); + assertThatThrownBy(() -> secondOrchestrator.acquire("operation-b", Duration.ZERO)) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, exception -> + assertThat(exception.code()).isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_OPERATION_CONFLICT)); + verify(second.sourceGuard, never()).fence(any(), any()); + firstLease.close(); + secondOrchestrator.acquire("operation-b", Duration.ZERO).close(); + } + + @Test + void validatesNegativeAndOverflowButAllowsZeroAndNewInstanceStartsIdle() { + Harness first = harness(); + when(first.sourceGuard.fence(eq("operation-a"), any())).thenReturn(first.sourceLease); + when(first.producerCoordinator.quiesce(eq("operation-a"), any())).thenReturn(first.producerLease); + when(first.writeCoordinator.acquire(eq("operation-a"), any())).thenReturn(first.writeLease); + MigrationMaintenanceLease lease = first.orchestrator().acquire("operation-a", Duration.ZERO); + assertThatThrownBy(() -> first.orchestrator().acquire("operation-b", Duration.ofSeconds(-1))) + .isInstanceOf(MigrationMaintenanceException.class); + assertThatThrownBy(() -> first.orchestrator().acquire("operation-b", Duration.ofSeconds(Long.MAX_VALUE))) + .isInstanceOf(MigrationMaintenanceException.class); + lease.close(); + + Harness restarted = harness(); + when(restarted.sourceGuard.fence(eq("operation-b"), any())).thenReturn(restarted.sourceLease); + when(restarted.producerCoordinator.quiesce(eq("operation-b"), any())).thenReturn(restarted.producerLease); + when(restarted.writeCoordinator.acquire(eq("operation-b"), any())).thenReturn(restarted.writeLease); + restarted.orchestrator().acquire("operation-b", Duration.ZERO).close(); + } + + @Test + void interruptClassificationKeepsInterruptBit() { + Harness harness = harness(); + when(harness.deploymentAuthority.acquire(eq("operation-a"), any())).thenAnswer(invocation -> { + Thread.currentThread().interrupt(); + throw MigrationMaintenanceException.interrupted(); + }); + + assertThatThrownBy(() -> harness.orchestrator().acquire("operation-a", Duration.ZERO)) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, exception -> + assertThat(exception.code()) + .isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_MAINTENANCE_INTERRUPTED)); + assertThat(Thread.interrupted()).isTrue(); + } + + private static Harness harness() { + Harness harness = new Harness( + Mockito.mock(DeploymentSingletonAuthority.class), + Mockito.mock(DeploymentSingletonLease.class), + Mockito.mock(MigrationSourceGuard.class), + Mockito.mock(MigrationSourceLease.class), + Mockito.mock(MetadataMaintenanceCoordinator.class), + Mockito.mock(MetadataMaintenanceLease.class), + Mockito.mock(MetadataWriteAdmissionCoordinator.class), + Mockito.mock(MetadataWriteMaintenanceLease.class)); + when(harness.deploymentAuthority.acquire(any(), any())).thenReturn(harness.authorityLease); + when(harness.producerCoordinator.snapshot()).thenReturn( + new MetadataMaintenanceSnapshot(MetadataMaintenancePhase.RUNNING, null, 0)); + return harness; + } + + private record Harness( + DeploymentSingletonAuthority deploymentAuthority, + DeploymentSingletonLease authorityLease, + MigrationSourceGuard sourceGuard, + MigrationSourceLease sourceLease, + MetadataMaintenanceCoordinator producerCoordinator, + MetadataMaintenanceLease producerLease, + MetadataWriteAdmissionCoordinator writeCoordinator, + MetadataWriteMaintenanceLease writeLease) { + + DefaultMigrationMaintenanceOrchestrator orchestrator() { + return new DefaultMigrationMaintenanceOrchestrator( + deploymentAuthority, sourceGuard, producerCoordinator, writeCoordinator); + } + } + + private static final class FakeSingletonAuthority implements DeploymentSingletonAuthority { + + private boolean owned; + + @Override + public synchronized DeploymentSingletonLease acquire(String operationId, Duration timeout) { + if (owned) { + throw MigrationMaintenanceException.operationConflict(); + } + owned = true; + return this::release; + } + + private synchronized void release() { + owned = false; + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/EmbeddedH2SourceClassifierTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/EmbeddedH2SourceClassifierTest.java new file mode 100644 index 0000000000..b26aa4ed3d --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/EmbeddedH2SourceClassifierTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +class EmbeddedH2SourceClassifierTest { + + @Test + void acceptsOnlyExplicitEmbeddedMemoryAndLocalFileUrls() { + assertThat(EmbeddedH2SourceClassifier.isSafeEmbeddedSource("H2", "jdbc:h2:mem:manager")).isTrue(); + assertThat(EmbeddedH2SourceClassifier.isSafeEmbeddedSource("H2", "jdbc:h2:file:/var/lib/manager")).isTrue(); + assertThat(EmbeddedH2SourceClassifier.isSafeEmbeddedSource("H2", "jdbc:h2:./data/manager")).isTrue(); + assertThat(EmbeddedH2SourceClassifier.isSafeEmbeddedSource("H2", "jdbc:h2:.\\data\\manager")).isTrue(); + assertThat(EmbeddedH2SourceClassifier.isSafeEmbeddedSource("H2", "jdbc:h2:..\\data\\manager")).isTrue(); + assertThat(EmbeddedH2SourceClassifier.isSafeEmbeddedSource("H2", "jdbc:h2:C:\\data\\manager")).isTrue(); + assertThat(EmbeddedH2SourceClassifier.isSafeEmbeddedSource("H2", "jdbc:h2:tcp://db/manager")).isFalse(); + assertThat(EmbeddedH2SourceClassifier.isSafeEmbeddedSource("H2", "jdbc:h2:ssl://db/manager")).isFalse(); + assertThat(EmbeddedH2SourceClassifier.isSafeEmbeddedSource("H2", "jdbc:h2:file:tcp://db/manager")).isFalse(); + assertThat(EmbeddedH2SourceClassifier.isSafeEmbeddedSource("H2", "jdbc:h2:file:////db/manager")).isFalse(); + assertThat(EmbeddedH2SourceClassifier.isSafeEmbeddedSource("H2", "jdbc:h2:\\\\db\\manager")).isFalse(); + assertThat(EmbeddedH2SourceClassifier.isSafeEmbeddedSource( + "H2", "jdbc:h2:file:/var/lib/manager;AUTO_SERVER=TRUE")).isFalse(); + assertThat(EmbeddedH2SourceClassifier.isSafeEmbeddedSource( + "H2", "jdbc:h2:file:/var/lib/manager;FILE_LOCK=NO")).isFalse(); + assertThat(EmbeddedH2SourceClassifier.isSafeEmbeddedSource("PostgreSQL", "jdbc:h2:mem:manager")).isFalse(); + assertThat(EmbeddedH2SourceClassifier.isSafeEmbeddedSource("H2", "jdbc:unknown:manager")).isFalse(); + } + + @Test + void matchesConfiguredSourceToActualMetadataLocation() { + assertThat(EmbeddedH2SourceClassifier.matchesConfiguredSource( + "jdbc:h2:mem:manager;MODE=MYSQL", "jdbc:h2:mem:manager")).isTrue(); + assertThat(EmbeddedH2SourceClassifier.matchesConfiguredSource( + "jdbc:h2:./data/manager;MODE=MYSQL", "jdbc:h2:file:./data/manager")).isTrue(); + assertThat(EmbeddedH2SourceClassifier.matchesConfiguredSource( + "jdbc:h2:mem:manager", "jdbc:h2:mem:other")).isFalse(); + assertThat(EmbeddedH2SourceClassifier.matchesConfiguredSource( + "jdbc:h2:file:/safe/manager", "jdbc:h2:file:/other/manager")).isFalse(); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/EmbeddedH2SourceGuardTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/EmbeddedH2SourceGuardTest.java new file mode 100644 index 0000000000..e84fde6a57 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/EmbeddedH2SourceGuardTest.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.DriverManager; +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import javax.sql.DataSource; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mockito; +import org.springframework.boot.jdbc.autoconfigure.DataSourceProperties; +import org.springframework.jdbc.datasource.DriverManagerDataSource; + +class EmbeddedH2SourceGuardTest { + + @Test + void holdsSafeSourceConnectionUntilLeaseClosesExactlyOnce() throws Exception { + DataSource dataSource = Mockito.mock(DataSource.class); + Connection connection = Mockito.mock(Connection.class); + DatabaseMetaData metadata = Mockito.mock(DatabaseMetaData.class); + when(dataSource.getConnection()).thenReturn(connection); + when(connection.getMetaData()).thenReturn(metadata); + when(metadata.getDatabaseProductName()).thenReturn("H2"); + when(metadata.getURL()).thenReturn("jdbc:h2:mem:manager"); + EmbeddedH2SourceGuard guard = guard(dataSource, "jdbc:h2:mem:manager"); + + MigrationSourceLease lease = guard.fence("operation-a", Duration.ofSeconds(1)); + + verify(connection, never()).close(); + lease.close(); + lease.close(); + verify(connection).close(); + guard.destroy(); + } + + @Test + void rejectsAmbiguousSourceAndClosesConnectionWithoutDetails() throws Exception { + DataSource dataSource = Mockito.mock(DataSource.class); + Connection connection = Mockito.mock(Connection.class); + DatabaseMetaData metadata = Mockito.mock(DatabaseMetaData.class); + when(dataSource.getConnection()).thenReturn(connection); + when(connection.getMetaData()).thenReturn(metadata); + when(metadata.getDatabaseProductName()).thenReturn("H2"); + when(metadata.getURL()).thenReturn("jdbc:h2:tcp://private-host/secret"); + EmbeddedH2SourceGuard guard = guard(dataSource, "jdbc:h2:mem:manager"); + + assertThatThrownBy(() -> guard.fence("operation-a", Duration.ofSeconds(1))) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, exception -> { + assertThat(exception.code()) + .isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_SOURCE_UNAVAILABLE); + assertThat(exception.safeMessage()).doesNotContain("private-host").doesNotContain("secret"); + assertThat(exception.getCause()).isNull(); + }); + verify(connection).close(); + guard.destroy(); + } + + @Test + void rejectsRawAutoServerAndFileLockSettingsThatMetadataRemoves(@TempDir Path directory) throws Exception { + String baseUrl = "jdbc:h2:file:" + directory.resolve("manager"); + EmbeddedH2SourceGuard safeGuard = guard(new DriverManagerDataSource(baseUrl), baseUrl); + safeGuard.fence("safe-source", Duration.ofSeconds(1)).close(); + safeGuard.destroy(); + for (String setting : new String[] {";AUTO_SERVER=TRUE", ";FILE_LOCK=NO"}) { + String configuredUrl = baseUrl + setting; + try (Connection connection = DriverManager.getConnection(configuredUrl)) { + assertThat(connection.getMetaData().getURL()).doesNotContain(setting.substring(1)); + } + DriverManagerDataSource dataSource = new DriverManagerDataSource(configuredUrl); + assertThatThrownBy(() -> guard(dataSource, configuredUrl) + .fence("operation-a", Duration.ofSeconds(1))) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, exception -> + assertThat(exception.code()) + .isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_SOURCE_UNAVAILABLE)); + } + } + + @Test + void timeoutAbandonsBlockingConnectionAndClosesLateResult() throws Exception { + assertTimedOutConnectionClosed(Duration.ZERO); + assertTimedOutConnectionClosed(Duration.ofMillis(20)); + } + + @Test + void rejectsOverflowDurationWithoutStartingConnectionWork() throws Exception { + DataSource dataSource = Mockito.mock(DataSource.class); + EmbeddedH2SourceGuard guard = guard(dataSource, "jdbc:h2:mem:manager"); + + assertThatThrownBy(() -> guard.fence("operation-a", Duration.ofSeconds(Long.MAX_VALUE))) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, exception -> + assertThat(exception.code()).isEqualTo(MigrationMaintenanceErrorCode.INVALID_REQUEST)); + verify(dataSource, never()).getConnection(); + guard.destroy(); + } + + @Test + void destroyRejectsNewAcquisitionWithoutStartingConnectionWork() throws Exception { + DataSource dataSource = Mockito.mock(DataSource.class); + EmbeddedH2SourceGuard guard = guard(dataSource, "jdbc:h2:mem:manager"); + guard.destroy(); + + assertThatThrownBy(() -> guard.fence("operation-a", Duration.ofSeconds(1))) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, exception -> + assertThat(exception.code()).isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_SOURCE_UNAVAILABLE)); + verify(dataSource, never()).getConnection(); + } + + private void assertTimedOutConnectionClosed(Duration timeout) throws Exception { + DataSource dataSource = Mockito.mock(DataSource.class); + Connection connection = Mockito.mock(Connection.class); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch closed = new CountDownLatch(1); + when(dataSource.getConnection()).thenAnswer(invocation -> { + entered.countDown(); + release.await(); + return connection; + }); + Mockito.doAnswer(invocation -> { + closed.countDown(); + return null; + }).when(connection).close(); + EmbeddedH2SourceGuard guard = guard(dataSource, "jdbc:h2:mem:manager"); + + assertThatThrownBy(() -> guard.fence("operation-a", timeout)) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, exception -> + assertThat(exception.code()) + .isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_MAINTENANCE_TIMEOUT)); + assertThat(entered.await(1, TimeUnit.SECONDS)).isTrue(); + release.countDown(); + assertThat(closed.await(1, TimeUnit.SECONDS)).isTrue(); + guard.destroy(); + } + + @Test + void interruptAbandonsBlockingConnectionAndPreservesInterrupt() throws Exception { + DataSource dataSource = Mockito.mock(DataSource.class); + Connection connection = Mockito.mock(Connection.class); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch closed = new CountDownLatch(1); + when(dataSource.getConnection()).thenAnswer(invocation -> { + entered.countDown(); + release.await(); + return connection; + }); + Mockito.doAnswer(invocation -> { + closed.countDown(); + return null; + }).when(connection).close(); + AtomicReference code = new AtomicReference<>(); + AtomicBoolean interrupted = new AtomicBoolean(); + EmbeddedH2SourceGuard guard = guard(dataSource, "jdbc:h2:mem:manager"); + Thread caller = Thread.ofPlatform().start(() -> { + try { + guard.fence("operation-a", Duration.ofSeconds(30)); + } catch (MigrationMaintenanceException exception) { + code.set(exception.code()); + interrupted.set(Thread.currentThread().isInterrupted()); + } + }); + assertThat(entered.await(1, TimeUnit.SECONDS)).isTrue(); + + caller.interrupt(); + caller.join(1_000); + assertThat(code.get()).isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_MAINTENANCE_INTERRUPTED); + assertThat(interrupted).isTrue(); + release.countDown(); + assertThat(closed.await(1, TimeUnit.SECONDS)).isTrue(); + guard.destroy(); + } + + @Test + void oneStuckDriverCallBoundsAllLaterAcquisitionWork() throws Exception { + DataSource dataSource = Mockito.mock(DataSource.class); + Connection connection = Mockito.mock(Connection.class); + AtomicInteger calls = new AtomicInteger(); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch closed = new CountDownLatch(1); + when(dataSource.getConnection()).thenAnswer(invocation -> { + calls.incrementAndGet(); + entered.countDown(); + release.await(); + return connection; + }); + Mockito.doAnswer(invocation -> { + closed.countDown(); + return null; + }).when(connection).close(); + EmbeddedH2SourceGuard guard = guard(dataSource, "jdbc:h2:mem:manager"); + + assertThatThrownBy(() -> guard.fence("operation-a", Duration.ZERO)) + .isInstanceOf(MigrationMaintenanceException.class); + assertThat(entered.await(1, TimeUnit.SECONDS)).isTrue(); + for (int index = 0; index < 20; index++) { + assertThatThrownBy(() -> guard.fence("operation-a", Duration.ZERO)) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, exception -> + assertThat(exception.code()) + .isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_MAINTENANCE_TIMEOUT)); + } + assertThat(calls).hasValue(1); + release.countDown(); + assertThat(closed.await(1, TimeUnit.SECONDS)).isTrue(); + guard.destroy(); + } + + private static EmbeddedH2SourceGuard guard(DataSource dataSource, String configuredUrl) { + DataSourceProperties properties = new DataSourceProperties(); + properties.setUrl(configuredUrl); + return new EmbeddedH2SourceGuard(dataSource, properties); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceCoordinatorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceCoordinatorTest.java index bcf2c54c0a..8ec7757007 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceCoordinatorTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/MetadataMaintenanceCoordinatorTest.java @@ -15,6 +15,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; @@ -199,7 +200,8 @@ class MetadataMaintenanceCoordinatorTest { } @Test - void failedParticipantRecoveryCannotBeReportedAsRunning() { + void failedParticipantRecoveryRequiresSameOperationAndCanBeRetriedExplicitly() { + AtomicBoolean resumeFails = new AtomicBoolean(true); MetadataMaintenanceParticipant participant = new MetadataMaintenanceParticipant() { @Override public String participantId() { @@ -213,7 +215,9 @@ class MetadataMaintenanceCoordinatorTest { @Override public void resume() { - throw MetadataMaintenanceException.resumeFailure(); + if (resumeFails.get()) { + throw MetadataMaintenanceException.resumeFailure(); + } } }; MetadataMaintenanceCoordinator coordinator = new MetadataMaintenanceCoordinator(List.of(participant)); @@ -223,6 +227,12 @@ class MetadataMaintenanceCoordinatorTest { assertThat(exception.code()).isEqualTo(MetadataMaintenanceErrorCode.QUIESCE_TIMEOUT)); assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataMaintenancePhase.RECOVERY_REQUIRED); + assertThatThrownBy(() -> coordinator.recover("operation-b")) + .isInstanceOfSatisfying(MetadataMaintenanceException.class, exception -> + assertThat(exception.code()).isEqualTo(MetadataMaintenanceErrorCode.OPERATION_CONFLICT)); + resumeFails.set(false); + coordinator.recover("operation-a"); + assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataMaintenancePhase.RUNNING); } @Test diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/MetadataWriteAdmissionStartupContextTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/MetadataWriteAdmissionStartupContextTest.java index 39d4876a75..253222d9f1 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/MetadataWriteAdmissionStartupContextTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/MetadataWriteAdmissionStartupContextTest.java @@ -8,8 +8,10 @@ package org.apache.hertzbeat.startup; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.lang.reflect.Method; +import java.time.Duration; import java.util.Arrays; import java.util.List; import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionAdvisor; @@ -21,6 +23,12 @@ import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceCoordinator; import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceParticipant; import org.apache.hertzbeat.manager.maintenance.AlertMetadataMaintenanceParticipant; import org.apache.hertzbeat.manager.maintenance.CollectorLifecycleMaintenanceParticipant; +import org.apache.hertzbeat.manager.maintenance.DeploymentSingletonAuthority; +import org.apache.hertzbeat.manager.maintenance.EmbeddedH2SourceGuard; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceOrchestrator; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceErrorCode; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceException; +import org.apache.hertzbeat.manager.maintenance.MigrationSourceGuard; import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; import org.apache.hertzbeat.warehouse.store.DataStorageDispatch; import org.apache.hertzbeat.warehouse.store.metadata.JdbcMonitorStatusMetadataWriter; @@ -52,6 +60,17 @@ class MetadataWriteAdmissionStartupContextTest { @Test void startupHasOneAdmissionBoundaryAndOneTransactionSource() throws Exception { assertThat(context.getBeansOfType(MetadataMaintenanceCoordinator.class)).hasSize(1); + assertThat(context.getBeansOfType(MigrationMaintenanceOrchestrator.class)).hasSize(1); + assertThat(context.getBeansOfType(MigrationSourceGuard.class)) + .containsOnlyKeys("embeddedH2SourceGuard"); + assertThat(context.getBean(MigrationSourceGuard.class)).isInstanceOf(EmbeddedH2SourceGuard.class); + assertThat(context.getBeansOfType(DeploymentSingletonAuthority.class)) + .containsOnlyKeys("unavailableDeploymentSingletonAuthority"); + assertThatThrownBy(() -> context.getBean(MigrationMaintenanceOrchestrator.class) + .acquire("normal-proof", Duration.ZERO)) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, exception -> + assertThat(exception.code()) + .isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_DEPLOYMENT_AUTHORITY_UNAVAILABLE)); List participants = context.getBeanProvider( MetadataMaintenanceParticipant.class).orderedStream().toList(); assertThat(participants) diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupRuntimeBoundaryContextTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupRuntimeBoundaryContextTest.java index d5df6cc69c..34b5871986 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupRuntimeBoundaryContextTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupRuntimeBoundaryContextTest.java @@ -30,10 +30,14 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.file.Path; import java.time.Clock; +import java.time.Duration; import org.apache.hertzbeat.common.runtime.BusinessRuntimeGate; import org.apache.hertzbeat.common.runtime.RuntimeMode; import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceErrorCode; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceException; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceOrchestrator; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupOperationState; import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; @@ -167,6 +171,15 @@ class StartupRuntimeBoundaryContextTest { assertTrue(context.containsBeanDefinition("periodicAlertRuleScheduler")); assertTrue(context.containsBeanDefinition("manageServer")); assertFalse(context.containsBeanDefinition("collectorLifecycleMaintenanceParticipant")); + MigrationMaintenanceOrchestrator orchestrator = + context.getBean(MigrationMaintenanceOrchestrator.class); + try { + orchestrator.acquire("gated-proof", Duration.ZERO); + org.junit.jupiter.api.Assertions.fail("Gated runtime must fail closed"); + } catch (MigrationMaintenanceException exception) { + assertEquals(MigrationMaintenanceErrorCode.MIGRATION_DEPLOYMENT_AUTHORITY_UNAVAILABLE, + exception.code()); + } assertTrue(context.containsBeanDefinition("otlpGrpcMetricsService")); assertTrue(context.containsBeanDefinition("alarmGroupReduce")); assertTrue(context.containsBeanDefinition("alarmInhibitReduce")); From 9c40cd466c3f93c4d3f5816a817abf7ea8875fdf Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 03:31:32 +0800 Subject: [PATCH 39/71] Coordinate standalone deployment ownership --- .../InstallationConvergenceVerifier.java | 15 ++ .../MigrationGuardConfiguration.java | 26 +++ ...NormalInstallationConvergenceVerifier.java | 34 ++++ .../StandaloneDeploymentOwnerView.java | 18 ++ ...tandaloneDeploymentSingletonAuthority.java | 87 +++++++++ .../setup/security/SecureSetupFile.java | 2 +- .../MigrationGuardConfigurationTest.java | 66 +++++++ ...aloneDeploymentSingletonAuthorityTest.java | 80 +++++++++ .../startup/HertzBeatApplication.java | 8 +- .../runtime/HertzBeatStartupCoordinator.java | 93 +++++++++- .../LocalInstallationStartupProbe.java | 6 + .../ResolvedStartupInstallationRoot.java | 14 ++ .../runtime/SpringStartupContextLauncher.java | 42 ++++- .../runtime/StandaloneDeploymentOwner.java | 170 ++++++++++++++++++ .../StandaloneDeploymentOwnerException.java | 20 +++ .../StandaloneDeploymentOwnerFactory.java | 19 ++ .../runtime/StandaloneFileStorePolicy.java | 25 +++ .../runtime/StartupContextLauncher.java | 11 ++ .../startup/runtime/StartupDecisionProbe.java | 6 + .../StartupInstallationRootResolver.java | 30 ++++ .../runtime/StartupModePropertyProbe.java | 8 + .../HertzBeatStartupCoordinatorTest.java | 133 ++++++++++++++ .../StandaloneDeploymentOwnerProcessMain.java | 33 ++++ .../StandaloneDeploymentOwnerTest.java | 135 ++++++++++++++ .../StandaloneFileStorePolicyTest.java | 28 +++ 25 files changed, 1099 insertions(+), 10 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/InstallationConvergenceVerifier.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/NormalInstallationConvergenceVerifier.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/StandaloneDeploymentOwnerView.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/StandaloneDeploymentSingletonAuthority.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/MigrationGuardConfigurationTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/StandaloneDeploymentSingletonAuthorityTest.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/ResolvedStartupInstallationRoot.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StandaloneDeploymentOwner.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StandaloneDeploymentOwnerException.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StandaloneDeploymentOwnerFactory.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StandaloneFileStorePolicy.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupInstallationRootResolver.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StandaloneDeploymentOwnerProcessMain.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StandaloneDeploymentOwnerTest.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StandaloneFileStorePolicyTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/InstallationConvergenceVerifier.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/InstallationConvergenceVerifier.java new file mode 100644 index 0000000000..b643057aae --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/InstallationConvergenceVerifier.java @@ -0,0 +1,15 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +/** Re-reads durable installation convergence for each migration fence. */ +@FunctionalInterface +public interface InstallationConvergenceVerifier { + + boolean isFullyConverged(); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationGuardConfiguration.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationGuardConfiguration.java index d2ba64220a..94b4ff9cd1 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationGuardConfiguration.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationGuardConfiguration.java @@ -7,6 +7,10 @@ package org.apache.hertzbeat.manager.maintenance; +import org.apache.hertzbeat.common.runtime.BusinessRuntimeGate; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; +import org.apache.hertzbeat.manager.setup.installation.InstallationRecordRepository; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -15,6 +19,28 @@ import org.springframework.context.annotation.Configuration; @Configuration(proxyBeanMethods = false) public class MigrationGuardConfiguration { + @Bean + @ConditionalOnNormalBusinessRuntime + @ConditionalOnBean(StandaloneDeploymentOwnerView.class) + @ConditionalOnMissingBean(InstallationConvergenceVerifier.class) + InstallationConvergenceVerifier normalInstallationConvergenceVerifier( + InstallationRecordRepository records, StandaloneDeploymentOwnerView owner) { + return new NormalInstallationConvergenceVerifier(records, owner); + } + + @Bean + @ConditionalOnBean({StandaloneDeploymentOwnerView.class, InstallationConvergenceVerifier.class}) + @ConditionalOnMissingBean(DeploymentSingletonAuthority.class) + DeploymentSingletonAuthority deploymentSingletonAuthority( + BusinessRuntimeGate runtimeGate, + StandaloneDeploymentOwnerView owner, + InstallationConvergenceVerifier convergence) { + if (runtimeGate.isOpen()) { + return new StandaloneDeploymentSingletonAuthority(owner, convergence); + } + return unavailableDeploymentSingletonAuthority(); + } + @Bean @ConditionalOnMissingBean(DeploymentSingletonAuthority.class) DeploymentSingletonAuthority unavailableDeploymentSingletonAuthority() { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/NormalInstallationConvergenceVerifier.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/NormalInstallationConvergenceVerifier.java new file mode 100644 index 0000000000..f38ce3488c --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/NormalInstallationConvergenceVerifier.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import java.nio.file.Path; +import org.apache.hertzbeat.manager.setup.installation.InstallationConvergenceService; +import org.apache.hertzbeat.manager.setup.installation.InstallationMode; +import org.apache.hertzbeat.manager.setup.installation.InstallationRecordRepository; + +/** Normal-context adapter that re-reads fingerprint and database installation state. */ +public final class NormalInstallationConvergenceVerifier implements InstallationConvergenceVerifier { + + private final InstallationRecordRepository records; + private final StandaloneDeploymentOwnerView owner; + + public NormalInstallationConvergenceVerifier( + InstallationRecordRepository records, StandaloneDeploymentOwnerView owner) { + this.records = records; + this.owner = owner; + } + + @Override + public boolean isFullyConverged() { + Path root = owner.installationRoot(); + return new InstallationConvergenceService( + records, root, root.resolve("data/config/.installation-fingerprint")).classify() + == InstallationMode.FULL; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/StandaloneDeploymentOwnerView.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/StandaloneDeploymentOwnerView.java new file mode 100644 index 0000000000..bdd8ad8a76 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/StandaloneDeploymentOwnerView.java @@ -0,0 +1,18 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import java.nio.file.Path; + +/** Non-owning view of the process-level standalone deployment lock. */ +public interface StandaloneDeploymentOwnerView { + + Path installationRoot(); + + boolean isValid(); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/StandaloneDeploymentSingletonAuthority.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/StandaloneDeploymentSingletonAuthority.java new file mode 100644 index 0000000000..7bd8597225 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/StandaloneDeploymentSingletonAuthority.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import java.time.Duration; +import java.util.concurrent.locks.ReentrantLock; + +/** Reserves migration operations beneath a live process-level standalone owner. */ +public final class StandaloneDeploymentSingletonAuthority implements DeploymentSingletonAuthority { + + private final ReentrantLock lock = new ReentrantLock(); + private final StandaloneDeploymentOwnerView owner; + private final InstallationConvergenceVerifier convergence; + private Object operationToken; + + public StandaloneDeploymentSingletonAuthority( + StandaloneDeploymentOwnerView owner, InstallationConvergenceVerifier convergence) { + this.owner = owner; + this.convergence = convergence; + } + + @Override + public DeploymentSingletonLease acquire(String operationId, Duration timeout) { + requireRequest(operationId, timeout); + lock.lock(); + try { + if (operationToken != null) { + throw MigrationMaintenanceException.operationConflict(); + } + if (!owner.isValid() || !convergence.isFullyConverged()) { + throw MigrationMaintenanceException.deploymentAuthorityUnavailable(); + } + Object token = new Object(); + operationToken = token; + return new OperationLease(token); + } catch (MigrationMaintenanceException exception) { + throw exception; + } catch (RuntimeException exception) { + throw MigrationMaintenanceException.deploymentAuthorityUnavailable(); + } finally { + lock.unlock(); + } + } + + private void requireRequest(String operationId, Duration timeout) { + if (operationId == null || operationId.isBlank() || timeout == null || timeout.isNegative()) { + throw MigrationMaintenanceException.invalidRequest(); + } + try { + timeout.toNanos(); + } catch (ArithmeticException exception) { + throw MigrationMaintenanceException.invalidRequest(); + } + } + + private final class OperationLease implements DeploymentSingletonLease { + + private final Object token; + private boolean closed; + + private OperationLease(Object token) { + this.token = token; + } + + @Override + public synchronized void close() { + if (closed) { + return; + } + lock.lock(); + try { + if (operationToken != token) { + throw MigrationMaintenanceException.operationConflict(); + } + operationToken = null; + closed = true; + } finally { + lock.unlock(); + } + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFile.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFile.java index a5772e52a1..47aed727f8 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFile.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFile.java @@ -127,7 +127,7 @@ public final class SecureSetupFile { return true; } - static Path prepareTrustedRoot(Path trustedRoot) throws IOException { + public static Path prepareTrustedRoot(Path trustedRoot) throws IOException { Path absoluteRoot = absolute(trustedRoot); if (!Files.exists(absoluteRoot, LinkOption.NOFOLLOW_LINKS)) { createMissingParents(absoluteRoot); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/MigrationGuardConfigurationTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/MigrationGuardConfigurationTest.java new file mode 100644 index 0000000000..d69cdcace6 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/MigrationGuardConfigurationTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +import java.nio.file.Path; +import org.apache.hertzbeat.common.runtime.BusinessRuntimeGate; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.setup.installation.InstallationRecordRepository; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; + +class MigrationGuardConfigurationTest { + + private final ApplicationContextRunner context = new ApplicationContextRunner() + .withPropertyValues(RuntimeMode.PROPERTY_NAME + "=" + RuntimeMode.NORMAL.value()) + .withUserConfiguration(MigrationGuardConfiguration.class) + .withBean(BusinessRuntimeGate.class, () -> BusinessRuntimeGate.fixed(RuntimeMode.NORMAL)) + .withBean(StandaloneDeploymentOwnerView.class, MigrationGuardConfigurationTest::owner) + .withBean(InstallationRecordRepository.class, () -> mock(InstallationRecordRepository.class)); + + @Test + void customConvergenceVerifierReplacesDefaultAndKeepsAuthorityUnique() { + InstallationConvergenceVerifier custom = () -> true; + + context.withBean(InstallationConvergenceVerifier.class, () -> custom).run(result -> { + assertThat(result).hasNotFailed(); + assertThat(result).hasSingleBean(InstallationConvergenceVerifier.class); + assertThat(result.getBean(InstallationConvergenceVerifier.class)).isSameAs(custom); + assertThat(result).hasSingleBean(DeploymentSingletonAuthority.class); + }); + } + + @Test + void customAuthorityReplacesStandaloneWhileDefaultVerifierRemainsUnique() { + DeploymentSingletonAuthority custom = (operationId, timeout) -> () -> { }; + + context.withBean(DeploymentSingletonAuthority.class, () -> custom).run(result -> { + assertThat(result).hasNotFailed(); + assertThat(result).hasSingleBean(DeploymentSingletonAuthority.class); + assertThat(result.getBean(DeploymentSingletonAuthority.class)).isSameAs(custom); + assertThat(result).hasSingleBean(InstallationConvergenceVerifier.class); + }); + } + + private static StandaloneDeploymentOwnerView owner() { + return new StandaloneDeploymentOwnerView() { + @Override + public Path installationRoot() { + return Path.of(".").toAbsolutePath().normalize(); + } + + @Override + public boolean isValid() { + return true; + } + }; + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/StandaloneDeploymentSingletonAuthorityTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/StandaloneDeploymentSingletonAuthorityTest.java new file mode 100644 index 0000000000..157249c0be --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/StandaloneDeploymentSingletonAuthorityTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.nio.file.Path; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; + +class StandaloneDeploymentSingletonAuthorityTest { + + @Test + void requiresLiveOwnerAndFreshFullConvergenceForEveryFence() { + AtomicBoolean valid = new AtomicBoolean(true); + AtomicBoolean full = new AtomicBoolean(false); + AtomicInteger checks = new AtomicInteger(); + StandaloneDeploymentOwnerView owner = ownerView(valid, checks); + StandaloneDeploymentSingletonAuthority authority = + new StandaloneDeploymentSingletonAuthority(owner, full::get); + + assertThatThrownBy(() -> authority.acquire("operation-a", Duration.ZERO)) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, exception -> + assertThat(exception.code()) + .isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_DEPLOYMENT_AUTHORITY_UNAVAILABLE)); + full.set(true); + DeploymentSingletonLease lease = authority.acquire("operation-a", Duration.ZERO); + lease.close(); + valid.set(false); + assertThatThrownBy(() -> authority.acquire("operation-b", Duration.ZERO)) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, exception -> + assertThat(exception.code()) + .isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_DEPLOYMENT_AUTHORITY_UNAVAILABLE)); + assertThat(checks).hasValue(3); + } + + @Test + void issuesOneOperationCapabilityAndCloseNeverClosesProcessOwner() { + AtomicBoolean valid = new AtomicBoolean(true); + AtomicInteger checks = new AtomicInteger(); + StandaloneDeploymentSingletonAuthority authority = new StandaloneDeploymentSingletonAuthority( + ownerView(valid, checks), () -> true); + + DeploymentSingletonLease lease = authority.acquire("operation-a", Duration.ZERO); + assertThatThrownBy(() -> authority.acquire("operation-a", Duration.ZERO)) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, exception -> + assertThat(exception.code()).isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_OPERATION_CONFLICT)); + assertThatThrownBy(() -> authority.acquire("operation-b", Duration.ZERO)) + .isInstanceOf(MigrationMaintenanceException.class); + lease.close(); + lease.close(); + authority.acquire("operation-b", Duration.ZERO).close(); + + assertThat(valid).isTrue(); + assertThat(checks).hasValue(2); + } + + private static StandaloneDeploymentOwnerView ownerView(AtomicBoolean valid, AtomicInteger checks) { + return new StandaloneDeploymentOwnerView() { + @Override + public Path installationRoot() { + return Path.of(".").toAbsolutePath().normalize(); + } + + @Override + public boolean isValid() { + checks.incrementAndGet(); + return valid.get(); + } + }; + } +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/HertzBeatApplication.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/HertzBeatApplication.java index 50bf2e5827..6555a4b79b 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/HertzBeatApplication.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/HertzBeatApplication.java @@ -24,6 +24,8 @@ import org.apache.hertzbeat.startup.runtime.HertzBeatStartupCoordinator; import org.apache.hertzbeat.startup.runtime.LocalInstallationStartupProbe; import org.apache.hertzbeat.startup.runtime.SpringStartupContextLauncher; import org.apache.hertzbeat.startup.runtime.StartupModePropertyProbe; +import org.apache.hertzbeat.startup.runtime.StartupInstallationRootResolver; +import org.apache.hertzbeat.startup.runtime.StandaloneDeploymentOwnerFactory; import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.context.properties.ConfigurationPropertiesScan; @@ -55,8 +57,12 @@ public class HertzBeatApplication { public static void main(String[] args) { SpringStartupContextLauncher launcher = new SpringStartupContextLauncher(); + StartupInstallationRootResolver rootResolver = new StartupInstallationRootResolver(); HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( - new StartupModePropertyProbe(new LocalInstallationStartupProbe()), launcher); + new StartupModePropertyProbe(new LocalInstallationStartupProbe()), launcher, + rootResolver, StandaloneDeploymentOwnerFactory.system()); + Runtime.getRuntime().addShutdownHook( + Thread.ofPlatform().name("hertzbeat-standalone-owner-close").unstarted(coordinator::close)); coordinator.start(args); } diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java index 7b1438f10b..d2ec6ef194 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java @@ -22,36 +22,70 @@ import org.apache.hertzbeat.common.runtime.RuntimeMode; import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; /** Serializes setup-to-normal transitions and always closes the old context first. */ -public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition { +public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition, AutoCloseable { private final StartupDecisionProbe probe; private final StartupContextLauncher launcher; private final StartupFailureReporter failureReporter; + private final StartupInstallationRootResolver rootResolver; + private final StandaloneDeploymentOwnerFactory ownerFactory; private String[] args = new String[0]; private RunningApplicationContext currentContext; + private ResolvedStartupInstallationRoot installationRoot; + private StandaloneDeploymentOwner deploymentOwner; private boolean normalRuntimeSelected; + private boolean convergenceConfirmed; + private boolean closed; public HertzBeatStartupCoordinator(StartupDecisionProbe probe, StartupContextLauncher launcher) { - this(probe, launcher, new StartupFailureReporter()); + this(probe, launcher, new StartupFailureReporter(), null, null); + } + + public HertzBeatStartupCoordinator( + StartupDecisionProbe probe, + StartupContextLauncher launcher, + StartupInstallationRootResolver rootResolver, + StandaloneDeploymentOwnerFactory ownerFactory) { + this(probe, launcher, new StartupFailureReporter(), rootResolver, ownerFactory); } HertzBeatStartupCoordinator( StartupDecisionProbe probe, StartupContextLauncher launcher, StartupFailureReporter failureReporter) { + this(probe, launcher, failureReporter, null, null); + } + + HertzBeatStartupCoordinator( + StartupDecisionProbe probe, + StartupContextLauncher launcher, + StartupFailureReporter failureReporter, + StartupInstallationRootResolver rootResolver, + StandaloneDeploymentOwnerFactory ownerFactory) { this.probe = Objects.requireNonNull(probe, "probe"); this.launcher = Objects.requireNonNull(launcher, "launcher"); this.failureReporter = Objects.requireNonNull(failureReporter, "failureReporter"); + this.rootResolver = rootResolver; + this.ownerFactory = ownerFactory; } public synchronized RunningApplicationContext start(String[] applicationArgs) { + if (closed) { + throw StandaloneDeploymentOwnerException.unavailable(); + } args = applicationArgs == null ? new String[0] : applicationArgs.clone(); + acquireDeploymentOwner(); StartupDecision decision; try { - decision = Objects.requireNonNull(probe.probe(args.clone()), "startup decision"); + decision = probeDecision(); } catch (RuntimeException exception) { failureReporter.report(StartupFailureReporter.Stage.STARTUP_PROBE, RuntimeMode.RECOVERY, exception); decision = StartupDecision.recovery(); } - return transition(decision); + try { + return transition(decision); + } catch (RuntimeException exception) { + releaseOwnerAfterFailedStart(); + throw exception; + } } @Override @@ -61,12 +95,16 @@ public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition } // The intent may be stale; the durable startup probe remains authoritative for the target mode. StartupDecision currentDecision = Objects.requireNonNull( - probe.probe(args.clone()), "startup decision"); + probeDecision(), "startup decision"); transition(currentDecision); } @Override public synchronized void completeSetup() { + if (currentContext == null || currentContext.mode() != RuntimeMode.FULL_SETUP_GATED) { + return; + } + convergenceConfirmed = true; transition(StartupDecision.normal()); } @@ -123,8 +161,51 @@ public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition } private RunningApplicationContext launch(StartupDecision decision) { + if (deploymentOwner == null) { + return Objects.requireNonNull( + launcher.launch(decision, args.clone(), this), + "startup context launcher returned null for " + decision.mode().value()); + } + boolean exposeAuthority = convergenceConfirmed && decision.mode() == RuntimeMode.NORMAL; return Objects.requireNonNull( - launcher.launch(decision, args.clone(), this), + launcher.launch(decision, args.clone(), this, installationRoot.canonicalRoot(), + exposeAuthority ? deploymentOwner.view() : null), "startup context launcher returned null for " + decision.mode().value()); } + + private StartupDecision probeDecision() { + StartupDecision decision = installationRoot == null + ? probe.probe(args.clone()) + : probe.probe(args.clone(), installationRoot.canonicalRoot()); + return Objects.requireNonNull(decision, "startup decision"); + } + + private void acquireDeploymentOwner() { + if (deploymentOwner != null || ownerFactory == null || rootResolver == null) { + return; + } + installationRoot = rootResolver.resolve(args.clone()); + deploymentOwner = Objects.requireNonNull( + ownerFactory.acquire(installationRoot), "standalone deployment owner"); + } + + private void releaseOwnerAfterFailedStart() { + if (currentContext == null && deploymentOwner != null) { + deploymentOwner.close(); + deploymentOwner = null; + } + } + + @Override + public synchronized void close() { + if (closed) { + return; + } + closed = true; + closeCurrent(); + if (deploymentOwner != null) { + deploymentOwner.close(); + deploymentOwner = null; + } + } } diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbe.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbe.java index 3ec7bb77ff..0251f2ba26 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbe.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/LocalInstallationStartupProbe.java @@ -49,6 +49,12 @@ public final class LocalInstallationStartupProbe implements StartupDecisionProbe @Override public StartupDecision probe(String[] args) { Path root = fixedRoot == null ? installationRoot(args) : fixedRoot; + return probe(args, root); + } + + @Override + public StartupDecision probe(String[] args, Path installationRoot) { + Path root = fixedRoot == null ? installationRoot : fixedRoot; boolean externalDatabaseConfigured = fixedExternalDatabaseConfigured == null ? externalDatabaseConfigured(args) : fixedExternalDatabaseConfigured; State managed = new ManagedActiveConfigurationInspector(root).inspect().state(); diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/ResolvedStartupInstallationRoot.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/ResolvedStartupInstallationRoot.java new file mode 100644 index 0000000000..9444a28039 --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/ResolvedStartupInstallationRoot.java @@ -0,0 +1,14 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.runtime; + +import java.nio.file.Path; + +/** One resolved startup root shared by ownership, probing, and Spring context launch. */ +record ResolvedStartupInstallationRoot(Path declaredRoot, Path canonicalRoot) { +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/SpringStartupContextLauncher.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/SpringStartupContextLauncher.java index e0f013ff87..fba1692de5 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/SpringStartupContextLauncher.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/SpringStartupContextLauncher.java @@ -17,7 +17,10 @@ package org.apache.hertzbeat.startup.runtime; +import java.nio.file.Path; import java.util.Map; +import org.apache.hertzbeat.manager.maintenance.StandaloneDeploymentOwnerView; +import org.apache.hertzbeat.manager.setup.config.SetupInstallationPaths; import org.apache.hertzbeat.bootstrap.SetupOnlyApplication; import org.apache.hertzbeat.common.runtime.RuntimeMode; import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; @@ -34,25 +37,60 @@ public final class SpringStartupContextLauncher implements StartupContextLaunche @Override public RunningApplicationContext launch( StartupDecision decision, String[] args, SetupRuntimeTransition setupRuntimeTransition) { - ConfigurableApplicationContext context = launchSpringContext(decision, args, setupRuntimeTransition); + ConfigurableApplicationContext context = launchSpringContext( + decision, args, setupRuntimeTransition, null, null); + return new SpringRunningApplicationContext(decision.mode(), context); + } + + @Override + public RunningApplicationContext launch( + StartupDecision decision, + String[] args, + SetupRuntimeTransition setupRuntimeTransition, + Path installationRoot, + StandaloneDeploymentOwnerView authorityView) { + ConfigurableApplicationContext context = launchSpringContext( + decision, args, setupRuntimeTransition, installationRoot, authorityView); return new SpringRunningApplicationContext(decision.mode(), context); } ConfigurableApplicationContext launchSpringContext( StartupDecision decision, String[] args, SetupRuntimeTransition setupRuntimeTransition) { + return launchSpringContext(decision, args, setupRuntimeTransition, null, null); + } + + private ConfigurableApplicationContext launchSpringContext( + StartupDecision decision, + String[] args, + SetupRuntimeTransition setupRuntimeTransition, + Path installationRoot, + StandaloneDeploymentOwnerView authorityView) { StandardEnvironment environment = new StandardEnvironment(); environment.getPropertySources().addFirst(new MapPropertySource( ManagedConfigEnvironmentPostProcessor.INTERNAL_RUNTIME_PROPERTY_SOURCE, - Map.of(RuntimeMode.PROPERTY_NAME, decision.mode().value()))); + internalProperties(decision, installationRoot))); return new SpringApplicationBuilder(sourceFor(decision.mode())) .environment(environment) .initializers(context -> { context.getBeanFactory().registerSingleton( "setupRuntimeTransition", setupRuntimeTransition); + if (decision.mode() == RuntimeMode.NORMAL && authorityView != null) { + context.getBeanFactory().registerSingleton( + "standaloneDeploymentOwnerView", authorityView); + } }) .run(args); } + private Map internalProperties(StartupDecision decision, Path installationRoot) { + if (installationRoot == null) { + return Map.of(RuntimeMode.PROPERTY_NAME, decision.mode().value()); + } + return Map.of( + RuntimeMode.PROPERTY_NAME, decision.mode().value(), + SetupInstallationPaths.ROOT_PROPERTY, installationRoot.toString()); + } + static Class sourceFor(RuntimeMode mode) { return mode == RuntimeMode.NORMAL || mode == RuntimeMode.FULL_SETUP_GATED ? HertzBeatApplication.class : SetupOnlyApplication.class; diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StandaloneDeploymentOwner.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StandaloneDeploymentOwner.java new file mode 100644 index 0000000000..95fa53686e --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StandaloneDeploymentOwner.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.runtime; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Objects; +import java.util.Set; +import org.apache.hertzbeat.manager.maintenance.StandaloneDeploymentOwnerView; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; + +/** Process-lifetime OS owner for one canonical standalone installation root. */ +public final class StandaloneDeploymentOwner implements AutoCloseable { + + static final String LOCK_PATH = "data/config/.standalone-deployment-owner.lock"; + private static final byte[] LOCK_CONTENT = "standalone-deployment-owner-v1\n" + .getBytes(StandardCharsets.UTF_8); + private final Path declaredRoot; + private final Path canonicalRoot; + private final Path lockPath; + private final Object rootFileKey; + private final Object lockFileKey; + private final FileChannel channel; + private final FileLock fileLock; + private final StandaloneDeploymentOwnerView view = new OwnerView(); + private boolean closed; + + private StandaloneDeploymentOwner( + ResolvedStartupInstallationRoot root, + Path lockPath, + Object rootFileKey, + Object lockFileKey, + FileChannel channel, + FileLock fileLock) { + declaredRoot = root.declaredRoot(); + canonicalRoot = root.canonicalRoot(); + this.lockPath = lockPath; + this.rootFileKey = rootFileKey; + this.lockFileKey = lockFileKey; + this.channel = channel; + this.fileLock = fileLock; + } + + static StandaloneDeploymentOwner acquire(ResolvedStartupInstallationRoot root) { + Path lockPath = root.canonicalRoot().resolve(LOCK_PATH); + FileChannel channel = null; + FileLock fileLock = null; + try { + requireLocalFileStore(root.canonicalRoot()); + initializeLockFile(root.canonicalRoot(), lockPath); + Object rootKey = fileKey(root.canonicalRoot()); + Object lockKey = fileKey(lockPath); + channel = FileChannel.open(lockPath, + Set.of(StandardOpenOption.READ, StandardOpenOption.WRITE, LinkOption.NOFOLLOW_LINKS)); + fileLock = channel.tryLock(); + if (fileLock == null) { + throw StandaloneDeploymentOwnerException.unavailable(); + } + return new StandaloneDeploymentOwner(root, lockPath, rootKey, lockKey, channel, fileLock); + } catch (IOException | RuntimeException exception) { + closeQuietly(fileLock, channel); + if (exception instanceof StandaloneDeploymentOwnerException ownerFailure) { + throw ownerFailure; + } + throw StandaloneDeploymentOwnerException.unavailable(); + } + } + + public StandaloneDeploymentOwnerView view() { + return view; + } + + public Path installationRoot() { + return canonicalRoot; + } + + public synchronized boolean isValid() { + if (closed || !fileLock.isValid()) { + return false; + } + try { + return declaredRoot.toRealPath().equals(canonicalRoot) + && Objects.equals(rootFileKey, fileKey(canonicalRoot)) + && !Files.isSymbolicLink(lockPath) + && SecureSetupFile.isOwnerOnlyRegularFile(lockPath) + && Objects.equals(lockFileKey, fileKey(lockPath)); + } catch (IOException | RuntimeException exception) { + return false; + } + } + + @Override + public synchronized void close() { + if (closed) { + return; + } + closed = true; + closeQuietly(fileLock, channel); + } + + private static void initializeLockFile(Path root, Path lockPath) throws IOException { + try { + SecureSetupFile.create(root, lockPath, LOCK_CONTENT); + } catch (FileAlreadyExistsException existing) { + // The lock inode is persistent and is never unlinked during normal shutdown. + } + if (!SecureSetupFile.existsInsideRootWithoutLinks(root, lockPath) + || !SecureSetupFile.isOwnerOnlyRegularFile(lockPath)) { + throw new IOException("Standalone deployment owner lock is invalid"); + } + SecureSetupFile.forceParentDirectoryIfSupported(root, lockPath); + } + + private static Object fileKey(Path path) throws IOException { + Object key = Files.readAttributes(path, BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS).fileKey(); + if (key == null) { + throw new IOException("Standalone deployment owner identity is unavailable"); + } + return key; + } + + private static void requireLocalFileStore(Path root) throws IOException { + if (!StandaloneFileStorePolicy.supportsProcessOwnership(Files.getFileStore(root).type())) { + throw new IOException("Standalone deployment owner requires a local file store"); + } + } + + private static void closeQuietly(FileLock lock, FileChannel channel) { + try { + if (lock != null) { + lock.close(); + } + } catch (IOException exception) { + // The channel close below remains the final OS-lock release attempt. + } + try { + if (channel != null) { + channel.close(); + } + } catch (IOException exception) { + // Startup/shutdown reports only stable ownership state. + } + } + + private final class OwnerView implements StandaloneDeploymentOwnerView { + + @Override + public Path installationRoot() { + return canonicalRoot; + } + + @Override + public boolean isValid() { + return StandaloneDeploymentOwner.this.isValid(); + } + } +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StandaloneDeploymentOwnerException.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StandaloneDeploymentOwnerException.java new file mode 100644 index 0000000000..4ba80871c2 --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StandaloneDeploymentOwnerException.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.runtime; + +/** Stable secret-free fatal startup failure for standalone deployment ownership. */ +public final class StandaloneDeploymentOwnerException extends RuntimeException { + + private StandaloneDeploymentOwnerException() { + super("Standalone deployment ownership is unavailable"); + } + + static StandaloneDeploymentOwnerException unavailable() { + return new StandaloneDeploymentOwnerException(); + } +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StandaloneDeploymentOwnerFactory.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StandaloneDeploymentOwnerFactory.java new file mode 100644 index 0000000000..0f78a867ac --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StandaloneDeploymentOwnerFactory.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.runtime; + +/** Acquires the fatal process-level owner before any startup probe or Spring context. */ +@FunctionalInterface +public interface StandaloneDeploymentOwnerFactory { + + StandaloneDeploymentOwner acquire(ResolvedStartupInstallationRoot root); + + static StandaloneDeploymentOwnerFactory system() { + return StandaloneDeploymentOwner::acquire; + } +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StandaloneFileStorePolicy.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StandaloneFileStorePolicy.java new file mode 100644 index 0000000000..692289d8fb --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StandaloneFileStorePolicy.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.runtime; + +import java.util.Locale; + +/** Rejects file-store types whose locks cannot prove local process exclusion. */ +final class StandaloneFileStorePolicy { + + private StandaloneFileStorePolicy() { + } + + static boolean supportsProcessOwnership(String fileStoreType) { + String type = fileStoreType.toLowerCase(Locale.ROOT); + return type.equals("apfs") || type.equals("hfs") || type.equals("hfs+") + || type.equals("xfs") || type.equals("btrfs") || type.equals("zfs") + || type.equals("ntfs") || type.equals("tmpfs") || type.equals("overlay") + || type.startsWith("ext"); + } +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupContextLauncher.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupContextLauncher.java index 7ed1882a02..a3f8d39b23 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupContextLauncher.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupContextLauncher.java @@ -17,6 +17,8 @@ package org.apache.hertzbeat.startup.runtime; +import java.nio.file.Path; +import org.apache.hertzbeat.manager.maintenance.StandaloneDeploymentOwnerView; import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; /** Opens exactly one source set for a classified runtime mode. */ @@ -25,4 +27,13 @@ public interface StartupContextLauncher { RunningApplicationContext launch( StartupDecision decision, String[] args, SetupRuntimeTransition setupRuntimeTransition); + + default RunningApplicationContext launch( + StartupDecision decision, + String[] args, + SetupRuntimeTransition setupRuntimeTransition, + Path installationRoot, + StandaloneDeploymentOwnerView authorityView) { + return launch(decision, args, setupRuntimeTransition); + } } diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupDecisionProbe.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupDecisionProbe.java index 9a8cf848b3..b8b0b935e4 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupDecisionProbe.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupDecisionProbe.java @@ -17,9 +17,15 @@ package org.apache.hertzbeat.startup.runtime; +import java.nio.file.Path; + /** Replaceable read-only source for the startup decision made before opening a context. */ @FunctionalInterface public interface StartupDecisionProbe { StartupDecision probe(String[] args); + + default StartupDecision probe(String[] args, Path installationRoot) { + return probe(args); + } } diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupInstallationRootResolver.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupInstallationRootResolver.java new file mode 100644 index 0000000000..880b28bea8 --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupInstallationRootResolver.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.runtime; + +import java.io.IOException; +import java.nio.file.Path; +import org.apache.hertzbeat.manager.setup.config.SetupInstallationPaths; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; + +/** Resolves and canonicalizes the official standalone installation root exactly once. */ +public final class StartupInstallationRootResolver { + + private static final String ROOT_ENVIRONMENT = "HERTZBEAT_INTERNAL_INSTALLATION_ROOT"; + + ResolvedStartupInstallationRoot resolve(String[] args) { + String configured = StartupArgumentProperties.resolve(args, SetupInstallationPaths.ROOT_PROPERTY, + System.getProperty(SetupInstallationPaths.ROOT_PROPERTY), System.getenv(ROOT_ENVIRONMENT)); + Path declared = Path.of(configured == null ? "." : configured).toAbsolutePath().normalize(); + try { + return new ResolvedStartupInstallationRoot(declared, SecureSetupFile.prepareTrustedRoot(declared)); + } catch (IOException | RuntimeException exception) { + throw StandaloneDeploymentOwnerException.unavailable(); + } + } +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbe.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbe.java index f974ad29c6..84b7a40d55 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbe.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupModePropertyProbe.java @@ -17,6 +17,7 @@ package org.apache.hertzbeat.startup.runtime; +import java.nio.file.Path; import java.util.Objects; import org.apache.hertzbeat.common.runtime.RuntimeMode; @@ -37,6 +38,13 @@ public final class StartupModePropertyProbe implements StartupDecisionProbe { return decide(args, System.getProperty(PROPERTY_NAME), System.getenv(ENVIRONMENT_NAME)); } + @Override + public StartupDecision probe(String[] args, Path installationRoot) { + String value = StartupArgumentProperties.resolve( + args, PROPERTY_NAME, System.getProperty(PROPERTY_NAME), System.getenv(ENVIRONMENT_NAME)); + return value == null ? fallback.probe(args, installationRoot) : decisionFor(value); + } + StartupDecision decide(String[] args, String systemValue, String environmentValue) { String value = StartupArgumentProperties.resolve(args, PROPERTY_NAME, systemValue, environmentValue); return value == null ? fallback.probe(args) : decisionFor(value); diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinatorTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinatorTest.java index bde5811bbc..519713a7ff 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinatorTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinatorTest.java @@ -19,14 +19,17 @@ package org.apache.hertzbeat.startup.runtime; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.maintenance.StandaloneDeploymentOwnerView; import org.apache.hertzbeat.manager.setup.config.SetupInstallationPaths; import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; import org.junit.jupiter.api.Test; @@ -179,6 +182,88 @@ class HertzBeatStartupCoordinatorTest { assertEquals("startup context launcher returned null for normal", failure.getSuppressed()[0].getMessage()); } + @Test + void officialStartupOwnsRootBeforeProbeAndFailsFatallyOnContention() { + StartupInstallationRootResolver resolver = new StartupInstallationRootResolver(); + String[] args = rootArgs(); + AtomicInteger probes = new AtomicInteger(); + OfficialRecordingLauncher launcher = new OfficialRecordingLauncher(); + try (StandaloneDeploymentOwner existing = StandaloneDeploymentOwner.acquire(resolver.resolve(args))) { + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( + ignored -> { + probes.incrementAndGet(); + return StartupDecision.normal(); + }, launcher, resolver, StandaloneDeploymentOwnerFactory.system()); + + assertThrows(StandaloneDeploymentOwnerException.class, () -> coordinator.start(args)); + assertEquals(0, probes.get()); + assertTrue(launcher.events.isEmpty()); + } + } + + @Test + void gatedCompletionKeepsOwnerAcrossContextSwitchAndExposesOnlyConfirmedNormalView() { + StartupInstallationRootResolver resolver = new StartupInstallationRootResolver(); + String[] args = rootArgs(); + OfficialRecordingLauncher launcher = new OfficialRecordingLauncher(); + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( + ignored -> new StartupDecision(RuntimeMode.FULL_SETUP_GATED), launcher, + resolver, StandaloneDeploymentOwnerFactory.system()); + + coordinator.start(args); + assertNull(launcher.views.getFirst()); + assertThrows(StandaloneDeploymentOwnerException.class, + () -> StandaloneDeploymentOwner.acquire(resolver.resolve(args))); + launcher.transitions.getFirst().completeSetup(); + + assertTrue(launcher.views.getLast().isValid()); + assertEquals(List.of("open:full_setup_gated", "close:full_setup_gated", "open:normal"), launcher.events); + assertThrows(StandaloneDeploymentOwnerException.class, + () -> StandaloneDeploymentOwner.acquire(resolver.resolve(args))); + coordinator.close(); + assertFalse(launcher.views.getLast().isValid()); + try (StandaloneDeploymentOwner ignored = StandaloneDeploymentOwner.acquire(resolver.resolve(args))) { + assertTrue(ignored.isValid()); + } + } + + @Test + void forcedNormalDoesNotReceiveAuthorityWithoutTrustedCompletion() { + OfficialRecordingLauncher launcher = new OfficialRecordingLauncher(); + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( + new StartupModePropertyProbe(ignored -> new StartupDecision(RuntimeMode.FULL_SETUP_GATED)), + launcher, new StartupInstallationRootResolver(), StandaloneDeploymentOwnerFactory.system()); + + coordinator.start(new String[] { + "--" + SetupInstallationPaths.ROOT_PROPERTY + "=" + installationRoot, + "--" + StartupModePropertyProbe.PROPERTY_NAME + "=" + RuntimeMode.NORMAL.value() + }); + + assertEquals(RuntimeMode.NORMAL, coordinator.mode()); + assertNull(launcher.views.getFirst()); + coordinator.close(); + } + + @Test + void terminalContextLaunchFailureReleasesOwnerWithoutWaitingForShutdownHook() { + StartupInstallationRootResolver resolver = new StartupInstallationRootResolver(); + String[] args = rootArgs(); + StartupContextLauncher failing = (decision, ignored, transition) -> { + throw new IllegalStateException("launch failed"); + }; + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( + ignored -> StartupDecision.normal(), failing, resolver, StandaloneDeploymentOwnerFactory.system()); + + assertThrows(IllegalStateException.class, () -> coordinator.start(args)); + try (StandaloneDeploymentOwner owner = StandaloneDeploymentOwner.acquire(resolver.resolve(args))) { + assertTrue(owner.isValid()); + } + } + + private String[] rootArgs() { + return new String[] {"--" + SetupInstallationPaths.ROOT_PROPERTY + "=" + installationRoot}; + } + private static void restoreSystemProperty(String name, String value) { if (value == null) { System.clearProperty(name); @@ -223,4 +308,52 @@ class HertzBeatStartupCoordinatorTest { }; } } + + private static final class OfficialRecordingLauncher implements StartupContextLauncher { + + private final List events = new ArrayList<>(); + private final List transitions = new ArrayList<>(); + private final List views = new ArrayList<>(); + + @Override + public RunningApplicationContext launch( + StartupDecision decision, String[] args, SetupRuntimeTransition setupRuntimeTransition) { + throw new AssertionError("Official startup must supply its resolved root"); + } + + @Override + public RunningApplicationContext launch( + StartupDecision decision, + String[] args, + SetupRuntimeTransition setupRuntimeTransition, + Path resolvedRoot, + StandaloneDeploymentOwnerView authorityView) { + assertTrue(resolvedRoot.isAbsolute()); + events.add("open:" + decision.mode().value()); + transitions.add(setupRuntimeTransition); + views.add(authorityView); + return new RunningApplicationContext() { + private boolean active = true; + + @Override + public RuntimeMode mode() { + return decision.mode(); + } + + @Override + public boolean isActive() { + return active; + } + + @Override + public void close() { + if (authorityView != null) { + assertTrue(authorityView.isValid()); + } + active = false; + events.add("close:" + decision.mode().value()); + } + }; + } + } } diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StandaloneDeploymentOwnerProcessMain.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StandaloneDeploymentOwnerProcessMain.java new file mode 100644 index 0000000000..130a3bc131 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StandaloneDeploymentOwnerProcessMain.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.runtime; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.file.Path; + +/** Child-JVM protocol fixture for deterministic OS owner-lock tests. */ +public final class StandaloneDeploymentOwnerProcessMain { + + private StandaloneDeploymentOwnerProcessMain() { + } + + public static void main(String[] args) throws Exception { + StartupInstallationRootResolver resolver = new StartupInstallationRootResolver(); + try (StandaloneDeploymentOwner ignored = StandaloneDeploymentOwner.acquire( + resolver.resolve(new String[] {"--hertzbeat.internal.installation-root=" + Path.of(args[0])}))) { + System.out.println("LOCKED"); + System.out.flush(); + new BufferedReader(new InputStreamReader(System.in)).readLine(); + } catch (StandaloneDeploymentOwnerException exception) { + System.out.println("FAILED"); + System.out.flush(); + System.exit(23); + } + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StandaloneDeploymentOwnerTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StandaloneDeploymentOwnerTest.java new file mode 100644 index 0000000000..93eb7ae272 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StandaloneDeploymentOwnerTest.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.runtime; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.TimeUnit; +import org.apache.hertzbeat.manager.maintenance.DeploymentSingletonLease; +import org.apache.hertzbeat.manager.maintenance.StandaloneDeploymentSingletonAuthority; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class StandaloneDeploymentOwnerTest { + + @TempDir + private Path temporaryDirectory; + + @Test + void sameCanonicalRootHasOneOwnerWhileDifferentRootsAreDifferentDeployments() throws Exception { + Path firstRoot = temporaryDirectory.resolve("first"); + Path alias = temporaryDirectory.resolve("alias"); + ResolvedStartupInstallationRoot first = resolve(firstRoot); + Files.createSymbolicLink(alias, first.canonicalRoot()); + StandaloneDeploymentOwner owner = StandaloneDeploymentOwner.acquire(first); + + assertThatThrownBy(() -> StandaloneDeploymentOwner.acquire(resolve(alias))) + .isInstanceOf(StandaloneDeploymentOwnerException.class); + try (StandaloneDeploymentOwner different = StandaloneDeploymentOwner.acquire( + resolve(temporaryDirectory.resolve("different")))) { + assertThat(different.isValid()).isTrue(); + } + owner.close(); + try (StandaloneDeploymentOwner reacquired = StandaloneDeploymentOwner.acquire(resolve(firstRoot))) { + assertThat(reacquired.isValid()).isTrue(); + } + assertThat(Files.exists(firstRoot.resolve(StandaloneDeploymentOwner.LOCK_PATH))).isTrue(); + } + + @Test + void lockOrRootIdentityReplacementInvalidatesViewWithoutReacquiring() throws Exception { + Path root = temporaryDirectory.resolve("replace"); + StandaloneDeploymentOwner owner = StandaloneDeploymentOwner.acquire(resolve(root)); + Path lock = root.resolve(StandaloneDeploymentOwner.LOCK_PATH); + Files.move(lock, lock.resolveSibling("owner.old")); + Files.createFile(lock); + + assertThat(owner.view().isValid()).isFalse(); + owner.close(); + + Path rootReplacement = temporaryDirectory.resolve("root-replacement"); + StandaloneDeploymentOwner replacedRootOwner = StandaloneDeploymentOwner.acquire(resolve(rootReplacement)); + Files.move(rootReplacement, temporaryDirectory.resolve("moved-root")); + Files.createDirectories(rootReplacement); + assertThat(replacedRootOwner.view().isValid()).isFalse(); + replacedRootOwner.close(); + } + + @Test + void childJvmContentionCloseAndCrashReleaseAreDeterministic() throws Exception { + Path root = temporaryDirectory.resolve("process"); + ChildOwner first = startChild(root); + assertThat(first.readStatus()).isEqualTo("LOCKED"); + ChildOwner blocked = startChild(root); + assertThat(blocked.readStatus()).isEqualTo("FAILED"); + assertThat(blocked.process.waitFor(5, TimeUnit.SECONDS)).isTrue(); + assertThat(blocked.process.exitValue()).isEqualTo(23); + + first.release(); + ChildOwner afterClose = startChild(root); + assertThat(afterClose.readStatus()).isEqualTo("LOCKED"); + afterClose.process.destroyForcibly(); + assertThat(afterClose.process.waitFor(5, TimeUnit.SECONDS)).isTrue(); + + ChildOwner afterCrash = startChild(root); + assertThat(afterCrash.readStatus()).isEqualTo("LOCKED"); + afterCrash.release(); + } + + @Test + void migrationLeaseCloseDoesNotReleaseProcessOwner() throws Exception { + Path root = temporaryDirectory.resolve("lease"); + try (StandaloneDeploymentOwner owner = StandaloneDeploymentOwner.acquire(resolve(root))) { + StandaloneDeploymentSingletonAuthority authority = + new StandaloneDeploymentSingletonAuthority(owner.view(), () -> true); + DeploymentSingletonLease lease = authority.acquire("operation", java.time.Duration.ZERO); + lease.close(); + + ChildOwner blocked = startChild(root); + assertThat(blocked.readStatus()).isEqualTo("FAILED"); + assertThat(blocked.process.waitFor(5, TimeUnit.SECONDS)).isTrue(); + assertThat(blocked.process.exitValue()).isEqualTo(23); + } + ChildOwner released = startChild(root); + assertThat(released.readStatus()).isEqualTo("LOCKED"); + released.release(); + } + + private ResolvedStartupInstallationRoot resolve(Path root) { + return new StartupInstallationRootResolver().resolve( + new String[] {"--hertzbeat.internal.installation-root=" + root}); + } + + private ChildOwner startChild(Path root) throws Exception { + String java = Path.of(System.getProperty("java.home"), "bin", "java").toString(); + Process process = new ProcessBuilder(java, "-cp", System.getProperty("java.class.path"), + StandaloneDeploymentOwnerProcessMain.class.getName(), root.toString()) + .redirectErrorStream(true) + .start(); + return new ChildOwner(process, new BufferedReader(new InputStreamReader(process.getInputStream()))); + } + + private record ChildOwner(Process process, BufferedReader output) { + + String readStatus() throws Exception { + return output.readLine(); + } + + void release() throws Exception { + process.getOutputStream().write('\n'); + process.getOutputStream().flush(); + assertThat(process.waitFor(5, TimeUnit.SECONDS)).isTrue(); + assertThat(process.exitValue()).isZero(); + } + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StandaloneFileStorePolicyTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StandaloneFileStorePolicyTest.java new file mode 100644 index 0000000000..42aedef1cc --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StandaloneFileStorePolicyTest.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.runtime; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class StandaloneFileStorePolicyTest { + + @ParameterizedTest + @ValueSource(strings = {"apfs", "hfs+", "ext4", "xfs", "btrfs", "zfs", "ntfs", "tmpfs", "overlay"}) + void acceptsLocalFileStores(String type) { + assertThat(StandaloneFileStorePolicy.supportsProcessOwnership(type)).isTrue(); + } + + @ParameterizedTest + @ValueSource(strings = {"nfs", "nfs4", "smbfs", "cifs", "sshfs", "afs", "9p", "fuse", "custom"}) + void rejectsNetworkFileStores(String type) { + assertThat(StandaloneFileStorePolicy.supportsProcessOwnership(type)).isFalse(); + } +} From 6588f08563fc1d3768722a87bdce02386b86a226 Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 05:37:05 +0800 Subject: [PATCH 40/71] Add JDBC metadata copy primitive --- .../setup/workflow/CanonicalTableDigest.java | 141 +++++++ .../setup/workflow/CanonicalValueEncoder.java | 155 ++++++++ .../setup/workflow/FlywaySchemaHistory.java | 41 +- .../setup/workflow/JdbcMetadataMigration.java | 258 +++++++++++++ .../workflow/JdbcTargetSchemaObjectState.java | 175 +++++++++ .../setup/workflow/JdbcTargetSchemaState.java | 43 ++- .../workflow/MetadataIdentityRepair.java | 129 +++++++ .../workflow/MetadataJdbcValueAdapter.java | 144 +++++++ .../workflow/MetadataMigrationErrorCode.java | 29 ++ .../workflow/MetadataMigrationException.java | 35 ++ .../MetadataMigrationProgressSink.java | 27 ++ .../workflow/MetadataMigrationSession.java | 224 +++++++++++ .../workflow/MetadataMigrationStage.java | 27 ++ .../setup/workflow/MetadataRowCopier.java | 109 ++++++ .../workflow/MetadataSchemaInventory.java | 251 +++++++++++++ .../workflow/MetadataTableDescriptor.java | 156 ++++++++ .../setup/workflow/MigrationDeadline.java | 64 ++++ .../setup/workflow/TargetSchemaContract.java | 13 +- .../JdbcMetadataMigrationContractTest.java | 140 +++++++ .../MetadataMigrationSessionTest.java | 142 +++++++ .../JdbcMetadataMigrationDatabaseTest.java | 353 ++++++++++++++++++ .../TargetSchemaProvisionerDatabaseTest.java | 33 ++ 22 files changed, 2668 insertions(+), 21 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/CanonicalTableDigest.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/CanonicalValueEncoder.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigration.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcTargetSchemaObjectState.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataIdentityRepair.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataJdbcValueAdapter.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationErrorCode.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationException.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationProgressSink.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationSession.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationStage.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataRowCopier.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataSchemaInventory.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataTableDescriptor.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationDeadline.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationContractTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationSessionTest.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationDatabaseTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/CanonicalTableDigest.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/CanonicalTableDigest.java new file mode 100644 index 0000000000..0bb5ae4558 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/CanonicalTableDigest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HexFormat; +import java.util.Objects; +import java.util.StringJoiner; +import java.util.stream.Collectors; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Computes a typed, length-framed SHA-256 digest in canonical primary-key order and bounded memory. */ +final class CanonicalTableDigest { + + private final MetadataJdbcValueAdapter values; + + CanonicalTableDigest(MetadataJdbcValueAdapter values) { + this.values = values; + } + + Digest digest( + Connection connection, + MetadataTableDescriptor actual, + MetadataTableDescriptor logical, + MetadataDatabaseKind kind, + MigrationDeadline deadline) throws SQLException { + MessageDigest tableDigest = sha256(); + long rowCount = 0; + String sql = selectSql(actual, logical, kind); + try (PreparedStatement statement = connection.prepareStatement(sql)) { + deadline.apply(statement); + statement.setFetchSize(128); + try (ResultSet rows = statement.executeQuery()) { + while (rows.next()) { + deadline.check(); + rowCount++; + for (int index = 0; index < logical.columns().size(); index++) { + MetadataTableDescriptor.Column logicalColumn = logical.columns().get(index); + MetadataTableDescriptor.Column actualColumn = actual.column(logicalColumn.name()); + Object value = values.read(rows, index + 1, actual.name(), actualColumn, kind); + byte[] frame = CanonicalValueEncoder.encode( + logical.name(), logicalColumn, actualColumn, value, kind); + tableDigest.update(frame); + } + } + } + } + return new Digest(rowCount, tableDigest.digest()); + } + + private String selectSql( + MetadataTableDescriptor table, + MetadataTableDescriptor logical, + MetadataDatabaseKind kind) { + StringJoiner columns = new StringJoiner(", "); + for (MetadataTableDescriptor.Column logicalColumn : logical.columns()) { + MetadataTableDescriptor.Column column = table.column(logicalColumn.name()); + String quoted = quote(column.name(), kind); + columns.add(values.selectExpression(quoted, table.name(), column, kind)); + } + String order = table.primaryKey().stream() + .map(primaryKey -> table.columns().stream() + .filter(column -> column.name().equals(primaryKey)) + .findFirst() + .orElseThrow()) + .map(column -> orderExpression(column, kind)) + .collect(Collectors.joining(", ")); + return "SELECT " + columns + " FROM " + quote(table.name(), kind) + " ORDER BY " + order; + } + + private static String orderExpression( + MetadataTableDescriptor.Column column, + MetadataDatabaseKind kind) { + String quoted = quote(column.name(), kind); + if (!CanonicalValueEncoder.isCharacter(column)) { + return quoted; + } + return switch (kind) { + case H2 -> "CAST(" + quoted + " AS VARBINARY)"; + case MYSQL -> "BINARY " + quoted; + case POSTGRESQL -> "convert_to(" + quoted + ", 'UTF8')"; + }; + } + + private static MessageDigest sha256() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("SHA-256 is unavailable"); + } + } + + static String quote(String identifier, MetadataDatabaseKind kind) { + return kind == MetadataDatabaseKind.MYSQL ? '`' + identifier + '`' : '"' + identifier + '"'; + } + + static final class Digest { + + private final long rowCount; + private final byte[] checksum; + + Digest(long rowCount, byte[] checksum) { + this.rowCount = rowCount; + this.checksum = checksum.clone(); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Digest that)) { + return false; + } + return rowCount == that.rowCount && MessageDigest.isEqual(checksum, that.checksum); + } + + @Override + public int hashCode() { + return Objects.hash(rowCount, HexFormat.of().formatHex(checksum)); + } + } + +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/CanonicalValueEncoder.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/CanonicalValueEncoder.java new file mode 100644 index 0000000000..b37988822c --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/CanonicalValueEncoder.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.sql.Date; +import java.sql.Time; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalAccessor; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Encodes one logical JDBC value into a typed, length-framed canonical byte sequence. */ +final class CanonicalValueEncoder { + + private CanonicalValueEncoder() { + } + + static byte[] encode( + String table, + MetadataTableDescriptor.Column logicalColumn, + MetadataTableDescriptor.Column actualColumn, + Object value, + MetadataDatabaseKind actualKind) { + if (value == null) { + return frame((byte) 0, new byte[0]); + } + if (logicalColumn.semanticType(MetadataDatabaseKind.H2, table).equals("boolean")) { + return frame((byte) 2, new byte[]{booleanValue(value) ? (byte) 1 : (byte) 0}); + } + if (logicalColumn.timestampWithTimeZone()) { + Instant instant = instant(value, actualColumn, actualKind).truncatedTo(ChronoUnit.MICROS); + return frame((byte) 7, utf8(instant.toString())); + } + if (logicalColumn.semanticType(MetadataDatabaseKind.H2, table).equals("timestamp")) { + LocalDateTime timestamp = localTimestamp(value).truncatedTo(ChronoUnit.MICROS); + return frame((byte) 7, utf8(timestamp.toString())); + } + if (value instanceof byte[] bytes) { + return frame((byte) 1, bytes); + } else if (value instanceof Boolean bool) { + return frame((byte) 2, new byte[]{bool ? (byte) 1 : (byte) 0}); + } else if (value instanceof BigDecimal decimal) { + return frame((byte) 3, utf8(decimal.toPlainString())); + } else if (value instanceof Byte || value instanceof Short + || value instanceof Integer || value instanceof Long) { + return frame((byte) 4, utf8(value.toString())); + } else if (value instanceof Float number) { + return frame((byte) 5, utf8(Float.toHexString(number))); + } else if (value instanceof Double number) { + return frame((byte) 6, utf8(Double.toHexString(number))); + } else if (value instanceof Date || value instanceof Time || value instanceof TemporalAccessor) { + return frame((byte) 8, utf8(value.toString())); + } + return frame(isCharacter(logicalColumn) ? (byte) 9 : (byte) 10, + utf8(value.toString())); + } + + static boolean isCharacter(MetadataTableDescriptor.Column column) { + return switch (column.jdbcType()) { + case Types.CHAR, Types.VARCHAR, Types.LONGVARCHAR, + Types.NCHAR, Types.NVARCHAR, Types.LONGNVARCHAR, + Types.CLOB, Types.NCLOB -> true; + default -> false; + }; + } + + private static Instant instant( + Object value, + MetadataTableDescriptor.Column actualColumn, + MetadataDatabaseKind actualKind) { + if (value instanceof OffsetDateTime timestamp) { + return timestamp.toInstant(); + } + if (value instanceof ZonedDateTime timestamp) { + return timestamp.toInstant(); + } + if (value instanceof Instant instant) { + return instant; + } + if (value instanceof Timestamp timestamp) { + return actualKind == MetadataDatabaseKind.MYSQL || !actualColumn.timestampWithTimeZone() + ? timestamp.toLocalDateTime().toInstant(ZoneOffset.UTC) + : timestamp.toInstant(); + } + if (value instanceof LocalDateTime timestamp) { + return timestamp.toInstant(ZoneOffset.UTC); + } + throw new MetadataMigrationException(MetadataMigrationErrorCode.VERIFICATION); + } + + private static boolean booleanValue(Object value) { + if (value instanceof Boolean bool) { + return bool; + } + if (value instanceof Number number) { + return number.longValue() != 0; + } + if (value instanceof byte[] bytes) { + return bytes.length > 0 && bytes[0] != 0; + } + return Boolean.parseBoolean(value.toString()); + } + + private static LocalDateTime localTimestamp(Object value) { + if (value instanceof LocalDateTime timestamp) { + return timestamp; + } + if (value instanceof Timestamp timestamp) { + return timestamp.toLocalDateTime(); + } + if (value instanceof OffsetDateTime timestamp) { + return timestamp.withOffsetSameInstant(ZoneOffset.UTC).toLocalDateTime(); + } + if (value instanceof ZonedDateTime timestamp) { + return timestamp.withZoneSameInstant(ZoneOffset.UTC).toLocalDateTime(); + } + throw new MetadataMigrationException(MetadataMigrationErrorCode.VERIFICATION); + } + + private static byte[] frame(byte type, byte[] value) { + ByteBuffer frame = ByteBuffer.allocate(1 + Integer.BYTES + value.length); + frame.put(type); + frame.putInt(value.length); + frame.put(value); + return frame.array(); + } + + private static byte[] utf8(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywaySchemaHistory.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywaySchemaHistory.java index d40378d321..470a4060f8 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywaySchemaHistory.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywaySchemaHistory.java @@ -39,28 +39,39 @@ final class FlywaySchemaHistory { } boolean isCurrent(Connection connection, TargetSchemaBaseline baseline) throws SQLException { + return isCurrent(connection, baseline, 0); + } + + boolean isCurrent( + Connection connection, TargetSchemaBaseline baseline, int queryTimeoutSeconds) throws SQLException { Set currentTables = currentBaselineTables(connection); if (!currentTables.contains(TABLE)) { return false; } String sql = "SELECT installed_rank, version, type, script, checksum, success FROM " + TABLE; - try (Statement statement = connection.createStatement(); ResultSet result = statement.executeQuery(sql)) { - if (!result.next()) { - throw unexpectedTargetState(); + try (Statement statement = connection.createStatement()) { + if (queryTimeoutSeconds > 0) { + statement.setQueryTimeout(queryTimeoutSeconds); } - boolean current = result.getInt("installed_rank") == 1 - && TargetSchemaBaseline.VERSION.equals(result.getString("version")) - && TargetSchemaBaseline.TYPE.equals(result.getString("type")) - && TargetSchemaBaseline.SCRIPT.equals(result.getString("script")) - && baseline.checksum() == result.getInt("checksum") - && !result.wasNull() - && result.getBoolean("success"); - if (!current || result.next() || !currentTables.contains(TargetSchemaContract.TABLE) - || !currentTables.containsAll(baseline.expectedTables()) - || !new TargetSchemaContract(kind).matches(connection, baseline.expectedTables())) { - throw unexpectedTargetState(); + try (ResultSet result = statement.executeQuery(sql)) { + if (!result.next()) { + throw unexpectedTargetState(); + } + boolean current = result.getInt("installed_rank") == 1 + && TargetSchemaBaseline.VERSION.equals(result.getString("version")) + && TargetSchemaBaseline.TYPE.equals(result.getString("type")) + && TargetSchemaBaseline.SCRIPT.equals(result.getString("script")) + && baseline.checksum() == result.getInt("checksum") + && !result.wasNull() + && result.getBoolean("success"); + if (!current || result.next() || !currentTables.contains(TargetSchemaContract.TABLE) + || !currentTables.containsAll(baseline.expectedTables()) + || !new TargetSchemaContract(kind) + .matches(connection, baseline.expectedTables(), queryTimeoutSeconds)) { + throw unexpectedTargetState(); + } + return true; } - return true; } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigration.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigration.java new file mode 100644 index 0000000000..c357f9223c --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigration.java @@ -0,0 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLTimeoutException; +import java.sql.Statement; +import java.time.Duration; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** + * Synchronous H2-to-external JDBC copy and verification primitive. + * + *

The timeout is a JDBC-statement budget, not a connection or socket deadline. The caller owns + * both connections and must configure their network timeouts and an outer abort watchdog. + */ +public final class JdbcMetadataMigration { + + private final MetadataJdbcValueAdapter values = new MetadataJdbcValueAdapter(); + private final MetadataRowCopier copier = new MetadataRowCopier(values); + private final CanonicalTableDigest digests = new CanonicalTableDigest(values); + private final MetadataIdentityRepair identities = new MetadataIdentityRepair(); + private final CopyCheckpoint copyCheckpoint; + + public JdbcMetadataMigration() { + this(CopyCheckpoint.NO_OP); + } + + JdbcMetadataMigration(CopyCheckpoint copyCheckpoint) { + this.copyCheckpoint = Objects.requireNonNull(copyCheckpoint, "copyCheckpoint"); + } + + public void migrate( + Connection source, + Connection target, + MetadataDatabaseKind targetKind, + Duration timeout, + MetadataMigrationProgressSink progress) { + Objects.requireNonNull(source, "source"); + Objects.requireNonNull(target, "target"); + Objects.requireNonNull(progress, "progress"); + requireTargetKind(targetKind); + MigrationDeadline deadline = new MigrationDeadline(timeout); + MetadataMigrationErrorCode phase = MetadataMigrationErrorCode.SCHEMA; + try (MetadataMigrationSession session = new MetadataMigrationSession(source, target)) { + session.begin(targetKind); + safeProgress(progress, MetadataMigrationStage.INSPECTING, 0); + TargetSchemaBaseline baseline = TargetSchemaBaseline.load(targetKind); + if (!new FlywaySchemaHistory(targetKind) + .isCurrent(target, baseline, deadline.remainingSeconds())) { + throw new SQLException("Target baseline is absent", "55000"); + } + MetadataSchemaInventory sourceSchema = MetadataSchemaInventory.capture( + source, baseline.expectedTables(), MetadataDatabaseKind.H2, deadline); + MetadataSchemaInventory targetSchema = MetadataSchemaInventory.capture( + target, baseline.expectedTables(), targetKind, deadline); + if (!sourceSchema.hasSamePortableShape(targetSchema)) { + throw new SQLException("Source and target application schemas differ", "55000"); + } + List order = sourceSchema.foreignKeyOrder(); + lockTargetTables(target, order, targetKind, deadline); + requireEmptyTarget(target, targetSchema, order, targetKind, deadline); + + phase = MetadataMigrationErrorCode.COPY; + safeProgress(progress, MetadataMigrationStage.COPYING, 10); + copyTables(source, target, targetSchema, order, targetKind, deadline, progress); + + phase = MetadataMigrationErrorCode.VERIFICATION; + safeProgress(progress, MetadataMigrationStage.VERIFYING, 65); + verifyTables(source, target, targetSchema, order, targetKind, deadline, progress); + + phase = MetadataMigrationErrorCode.SEQUENCE; + safeProgress(progress, MetadataMigrationStage.REPAIRING, 90); + repairIdentities(target, targetSchema, order, targetKind, deadline); + deadline.check(); + session.commit(); + } catch (MetadataMigrationException exception) { + MetadataMigrationException cleanupFailure = cleanupFailure(exception); + if (cleanupFailure != null) { + throw cleanupFailure; + } + throw exception; + } catch (IOException | SQLException | RuntimeException exception) { + MetadataMigrationException cleanupFailure = cleanupFailure(exception); + if (cleanupFailure != null) { + throw cleanupFailure; + } + throw new MetadataMigrationException(isTimeout(exception) ? MetadataMigrationErrorCode.TIMEOUT : phase); + } + safeProgress(progress, MetadataMigrationStage.COMPLETE, 100); + } + + private void copyTables( + Connection source, + Connection target, + MetadataSchemaInventory targetSchema, + List order, + MetadataDatabaseKind targetKind, + MigrationDeadline deadline, + MetadataMigrationProgressSink progress) throws SQLException { + for (int index = 0; index < order.size(); index++) { + MetadataTableDescriptor sourceTable = order.get(index); + copier.copy( + source, + target, + sourceTable, + targetSchema.table(sourceTable.name()), + targetKind, + deadline); + copyCheckpoint.afterTable(sourceTable.name()); + safeProgress(progress, MetadataMigrationStage.COPYING, percentage(10, 60, index + 1, order.size())); + } + } + + private void verifyTables( + Connection source, + Connection target, + MetadataSchemaInventory targetSchema, + List order, + MetadataDatabaseKind targetKind, + MigrationDeadline deadline, + MetadataMigrationProgressSink progress) throws SQLException { + for (int index = 0; index < order.size(); index++) { + MetadataTableDescriptor sourceTable = order.get(index); + CanonicalTableDigest.Digest sourceDigest = digests.digest( + source, sourceTable, sourceTable, MetadataDatabaseKind.H2, deadline); + CanonicalTableDigest.Digest targetDigest = digests.digest( + target, targetSchema.table(sourceTable.name()), sourceTable, targetKind, deadline); + if (!sourceDigest.equals(targetDigest)) { + throw new SQLException("Copied metadata differs", "55000"); + } + safeProgress(progress, MetadataMigrationStage.VERIFYING, percentage(65, 88, index + 1, order.size())); + } + } + + private void repairIdentities( + Connection target, + MetadataSchemaInventory targetSchema, + List order, + MetadataDatabaseKind targetKind, + MigrationDeadline deadline) throws SQLException { + for (MetadataTableDescriptor sourceTable : order) { + identities.repair(target, targetSchema.table(sourceTable.name()), targetKind, deadline); + } + } + + private static void requireEmptyTarget( + Connection target, + MetadataSchemaInventory targetSchema, + List order, + MetadataDatabaseKind targetKind, + MigrationDeadline deadline) throws SQLException { + for (MetadataTableDescriptor sourceTable : order) { + MetadataTableDescriptor targetTable = targetSchema.table(sourceTable.name()); + String sql = "SELECT 1 FROM " + CanonicalTableDigest.quote(targetTable.name(), targetKind); + try (PreparedStatement statement = target.prepareStatement(sql)) { + deadline.apply(statement); + statement.setMaxRows(1); + try (ResultSet rows = statement.executeQuery()) { + if (rows.next()) { + throw new SQLException("Target application tables are not empty", "55000"); + } + } + } + } + } + + private static void lockTargetTables( + Connection target, + List order, + MetadataDatabaseKind targetKind, + MigrationDeadline deadline) throws SQLException { + if (targetKind != MetadataDatabaseKind.POSTGRESQL) { + return; + } + String tables = order.stream() + .map(MetadataTableDescriptor::name) + .map(table -> CanonicalTableDigest.quote(table, targetKind)) + .collect(Collectors.joining(", ")); + try (Statement statement = target.createStatement()) { + deadline.apply(statement); + statement.execute("LOCK TABLE " + tables + " IN SHARE ROW EXCLUSIVE MODE"); + } + } + + private static void safeProgress( + MetadataMigrationProgressSink progress, + MetadataMigrationStage stage, + int percent) { + try { + progress.report(stage, percent); + } catch (RuntimeException ignored) { + // Observability must not influence transactional correctness. + } + } + + private static int percentage(int start, int end, int completed, int total) { + return start + Math.floorDiv((end - start) * completed, total); + } + + private static void requireTargetKind(MetadataDatabaseKind targetKind) { + if (targetKind != MetadataDatabaseKind.MYSQL && targetKind != MetadataDatabaseKind.POSTGRESQL) { + throw new MetadataMigrationException(MetadataMigrationErrorCode.SCHEMA); + } + } + + @FunctionalInterface + interface CopyCheckpoint { + + CopyCheckpoint NO_OP = table -> { }; + + void afterTable(String table) throws SQLException; + } + + private static boolean isTimeout(Exception exception) { + if (exception instanceof SQLTimeoutException) { + return true; + } + if (exception instanceof SQLException sqlException) { + return "57014".equals(sqlException.getSQLState()) + || "HYT00".equals(sqlException.getSQLState()) + || "HYT01".equals(sqlException.getSQLState()); + } + return false; + } + + static MetadataMigrationException cleanupFailure(Exception exception) { + for (Throwable suppressed : exception.getSuppressed()) { + if (suppressed instanceof MetadataMigrationException migrationException + && migrationException.code() == MetadataMigrationErrorCode.ROLLBACK_OUTCOME_UNKNOWN) { + return migrationException; + } + } + return null; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcTargetSchemaObjectState.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcTargetSchemaObjectState.java new file mode 100644 index 0000000000..5392fa9afb --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcTargetSchemaObjectState.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Captures vendor-catalog objects that can change copy or identity behavior. */ +final class JdbcTargetSchemaObjectState { + + private JdbcTargetSchemaObjectState() { + } + + static void capture( + Connection connection, + MetadataDatabaseKind kind, + Set baselineTables, + int queryTimeoutSeconds, + FactSink facts) throws SQLException { + captureChecks(connection, kind, baselineTables, queryTimeoutSeconds, facts); + captureTriggers(connection, kind, baselineTables, queryTimeoutSeconds, facts); + captureSequences(connection, kind, queryTimeoutSeconds, facts); + } + + static void captureIdentityOwnership( + Connection connection, + MetadataDatabaseKind kind, + String table, + List identities, + int queryTimeoutSeconds, + FactSink facts) throws SQLException { + if (kind != MetadataDatabaseKind.POSTGRESQL) { + return; + } + try (PreparedStatement statement = connection.prepareStatement("SELECT pg_get_serial_sequence(?, ?)")) { + applyTimeout(statement, queryTimeoutSeconds); + for (String column : identities) { + statement.setString(1, table); + statement.setString(2, column); + try (ResultSet rows = statement.executeQuery()) { + if (!rows.next()) { + throw new SQLException("Target identity sequence ownership is absent", "55000"); + } + String sequence = rows.getString(1); + if (sequence == null) { + throw new SQLException("Target identity sequence ownership is absent", "55000"); + } + facts.add("identity-sequence", table, column, normalize(sequence)); + } + } + } + } + + private static void captureChecks( + Connection connection, + MetadataDatabaseKind kind, + Set baselineTables, + int queryTimeoutSeconds, + FactSink facts) throws SQLException { + String sql = kind == MetadataDatabaseKind.POSTGRESQL + ? "SELECT relation.relname, pg_get_constraintdef(c.oid, false) " + + "FROM pg_constraint c " + + "JOIN pg_class relation ON relation.oid = c.conrelid " + + "JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace " + + "WHERE c.contype = 'c' AND namespace.nspname = current_schema()" + : "SELECT table_constraint.table_name, check_constraint.check_clause " + + "FROM information_schema.table_constraints table_constraint " + + "JOIN information_schema.check_constraints check_constraint " + + "ON check_constraint.constraint_schema = table_constraint.constraint_schema " + + "AND check_constraint.constraint_name = table_constraint.constraint_name " + + "WHERE table_constraint.constraint_type = 'CHECK' " + + "AND table_constraint.table_schema = DATABASE()"; + try (PreparedStatement statement = connection.prepareStatement(sql)) { + applyTimeout(statement, queryTimeoutSeconds); + try (ResultSet rows = statement.executeQuery()) { + while (rows.next()) { + String table = normalize(rows.getString(1)); + if (baselineTables.contains(table)) { + facts.add("check", table, rows.getString(2).strip()); + } + } + } + } + } + + private static void captureTriggers( + Connection connection, + MetadataDatabaseKind kind, + Set baselineTables, + int queryTimeoutSeconds, + FactSink facts) throws SQLException { + String schemaPredicate = kind == MetadataDatabaseKind.POSTGRESQL + ? "trigger_schema = current_schema()" + : "trigger_schema = DATABASE()"; + String sql = "SELECT event_object_table, action_timing, event_manipulation, action_statement " + + "FROM information_schema.triggers WHERE " + schemaPredicate; + try (PreparedStatement statement = connection.prepareStatement(sql)) { + applyTimeout(statement, queryTimeoutSeconds); + try (ResultSet rows = statement.executeQuery()) { + while (rows.next()) { + String table = normalize(rows.getString(1)); + if (baselineTables.contains(table)) { + facts.add("trigger", table, normalize(rows.getString(2)), + normalize(rows.getString(3)), rows.getString(4).strip()); + } + } + } + } + } + + private static void captureSequences( + Connection connection, + MetadataDatabaseKind kind, + int queryTimeoutSeconds, + FactSink facts) throws SQLException { + if (kind != MetadataDatabaseKind.POSTGRESQL) { + return; + } + String sql = "SELECT schemaname, sequencename, increment_by, min_value, max_value, " + + "cache_size, cycle FROM pg_sequences WHERE schemaname = current_schema()"; + try (PreparedStatement statement = connection.prepareStatement(sql)) { + applyTimeout(statement, queryTimeoutSeconds); + try (ResultSet rows = statement.executeQuery()) { + while (rows.next()) { + String qualifiedName = normalize(rows.getString(1)) + '.' + normalize(rows.getString(2)); + facts.add( + "sequence", + qualifiedName, + "increment=" + rows.getLong(3), + "minimum=" + rows.getLong(4), + "maximum=" + rows.getLong(5), + "cache=" + rows.getLong(6), + "cycle=" + rows.getBoolean(7)); + } + } + } + } + + private static void applyTimeout(PreparedStatement statement, int queryTimeoutSeconds) throws SQLException { + if (queryTimeoutSeconds > 0) { + statement.setQueryTimeout(queryTimeoutSeconds); + } + } + + private static String normalize(String value) { + return value.toLowerCase(Locale.ROOT); + } + + @FunctionalInterface + interface FactSink { + + void add(String... parts); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcTargetSchemaState.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcTargetSchemaState.java index e8dd12de4d..94c3ae2792 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcTargetSchemaState.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcTargetSchemaState.java @@ -22,7 +22,9 @@ import java.sql.DatabaseMetaData; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Types; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; @@ -39,40 +41,61 @@ final class JdbcTargetSchemaState { Connection connection, MetadataDatabaseKind kind, Set baselineTables) throws SQLException { + return capture(connection, kind, baselineTables, 0); + } + + static SchemaState capture( + Connection connection, + MetadataDatabaseKind kind, + Set baselineTables, + int queryTimeoutSeconds) throws SQLException { DatabaseMetaData metadata = connection.getMetaData(); String catalog = connection.getCatalog(); String schema = kind == MetadataDatabaseKind.POSTGRESQL ? connection.getSchema() : null; FactCollector facts = new FactCollector(); for (String table : baselineTables.stream().sorted().toList()) { facts.add("table", table); - readColumns(metadata, catalog, schema, table, kind, facts); + List identities = readColumns(metadata, catalog, schema, table, kind, facts); + JdbcTargetSchemaObjectState.captureIdentityOwnership( + connection, kind, table, identities, queryTimeoutSeconds, facts::add); readPrimaryKey(metadata, catalog, schema, table, facts); readIndexes(metadata, catalog, schema, table, facts); readForeignKeys(metadata, catalog, schema, table, facts); } + JdbcTargetSchemaObjectState.capture( + connection, kind, baselineTables, queryTimeoutSeconds, facts::add); return facts.build(); } - private static void readColumns( + private static List readColumns( DatabaseMetaData metadata, String catalog, String schema, String table, MetadataDatabaseKind kind, FactCollector facts) throws SQLException { + List identities = new ArrayList<>(); try (ResultSet columns = metadata.getColumns(catalog, schema, table, null)) { while (columns.next()) { int jdbcType = columns.getInt("DATA_TYPE"); int size = columns.getInt("COLUMN_SIZE"); int scale = columns.getInt("DECIMAL_DIGITS"); + String column = normalize(columns.getString("COLUMN_NAME")); + String generated = generated(columns.getString("IS_AUTOINCREMENT")); + if (generated.equals("identity")) { + identities.add(column); + } facts.add( "column", table, - normalize(columns.getString("COLUMN_NAME")), + column, stableTypeFamily(kind, jdbcType, size, scale), - nullable(columns.getInt("NULLABLE"))); + nullable(columns.getInt("NULLABLE")), + generated, + defaultValue(columns.getString("COLUMN_DEF"))); } } + return List.copyOf(identities); } private static void readPrimaryKey( @@ -191,6 +214,18 @@ final class JdbcTargetSchemaState { }; } + private static String generated(String value) throws SQLException { + return switch (normalize(value)) { + case "yes" -> "identity"; + case "no", "" -> "not-identity"; + default -> throw new SQLException("Target schema identity metadata is unknown", "55000"); + }; + } + + private static String defaultValue(String value) { + return value == null ? "no-default" : "default=" + value.strip(); + } + private static String foreignKeyRule(short value) throws SQLException { return switch (value) { case DatabaseMetaData.importedKeyCascade -> "cascade"; diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataIdentityRepair.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataIdentityRepair.java new file mode 100644 index 0000000000..b117ad0660 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataIdentityRepair.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Repairs and proves vendor identity generators after explicit identifier insertion. */ +final class MetadataIdentityRepair { + + void repair( + Connection target, + MetadataTableDescriptor table, + MetadataDatabaseKind kind, + MigrationDeadline deadline) throws SQLException { + for (MetadataTableDescriptor.Column column : table.identityColumns()) { + Long maximum = maximum(target, table, column, kind, deadline); + if (maximum == null) { + continue; + } + if (kind == MetadataDatabaseKind.POSTGRESQL) { + repairPostgresql(target, table, column, maximum, deadline); + } else if (kind == MetadataDatabaseKind.MYSQL) { + verifyMysql(target, table, maximum, deadline); + } else { + throw new SQLException("Unsupported identity target", "55000"); + } + } + } + + private static Long maximum( + Connection connection, + MetadataTableDescriptor table, + MetadataTableDescriptor.Column column, + MetadataDatabaseKind kind, + MigrationDeadline deadline) throws SQLException { + String sql = "SELECT MAX(" + CanonicalTableDigest.quote(column.name(), kind) + ") FROM " + + CanonicalTableDigest.quote(table.name(), kind); + try (PreparedStatement statement = connection.prepareStatement(sql)) { + deadline.apply(statement); + try (ResultSet result = statement.executeQuery()) { + if (!result.next()) { + throw new SQLException("Identity maximum is unavailable", "55000"); + } + long maximum = result.getLong(1); + return result.wasNull() ? null : maximum; + } + } + } + + private static void repairPostgresql( + Connection connection, + MetadataTableDescriptor table, + MetadataTableDescriptor.Column column, + long maximum, + MigrationDeadline deadline) throws SQLException { + String sequence; + try (PreparedStatement statement = connection.prepareStatement("SELECT pg_get_serial_sequence(?, ?)")) { + deadline.apply(statement); + statement.setString(1, table.name()); + statement.setString(2, column.name()); + try (ResultSet result = statement.executeQuery()) { + if (!result.next() || (sequence = result.getString(1)) == null) { + throw new SQLException("Identity sequence is unavailable", "55000"); + } + } + } + try (PreparedStatement statement = + connection.prepareStatement("SELECT setval(CAST(? AS regclass), ?, true)")) { + deadline.apply(statement); + statement.setString(1, sequence); + statement.setLong(2, maximum); + statement.executeQuery().close(); + } + try (PreparedStatement statement = + connection.prepareStatement("SELECT nextval(CAST(? AS regclass))")) { + deadline.apply(statement); + statement.setString(1, sequence); + try (ResultSet result = statement.executeQuery()) { + if (!result.next() || result.getLong(1) <= maximum) { + throw new SQLException("Identity sequence did not advance", "55000"); + } + } + } + try (PreparedStatement statement = + connection.prepareStatement("SELECT setval(CAST(? AS regclass), ?, true)")) { + deadline.apply(statement); + statement.setString(1, sequence); + statement.setLong(2, maximum); + statement.executeQuery().close(); + } + } + + private static void verifyMysql( + Connection connection, + MetadataTableDescriptor table, + long maximum, + MigrationDeadline deadline) throws SQLException { + String sql = "SELECT AUTO_INCREMENT FROM information_schema.tables " + + "WHERE table_schema = DATABASE() AND table_name = ?"; + try (PreparedStatement statement = connection.prepareStatement(sql)) { + deadline.apply(statement); + statement.setString(1, table.name()); + try (ResultSet result = statement.executeQuery()) { + if (!result.next() || result.getLong(1) <= maximum || result.wasNull()) { + throw new SQLException("Identity counter did not advance", "55000"); + } + } + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataJdbcValueAdapter.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataJdbcValueAdapter.java new file mode 100644 index 0000000000..e093bd83a1 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataJdbcValueAdapter.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.IOException; +import java.io.Reader; +import java.nio.charset.StandardCharsets; +import java.sql.Blob; +import java.sql.Clob; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLXML; +import java.sql.Types; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.Set; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Materializes JDBC values and adapts the three PostgreSQL large-object text columns. */ +final class MetadataJdbcValueAdapter { + + private static final Set POSTGRESQL_OID_TEXT = Set.of( + "hzb_ai_message.content", "hzb_define.content", "hzb_notice_template.content"); + + Object read( + ResultSet rows, + int index, + String table, + MetadataTableDescriptor.Column column, + MetadataDatabaseKind kind) throws SQLException { + Object value = rows.getObject(index); + if (value == null) { + return null; + } + if (isPostgresqlOidText(kind, table, column.name()) && value instanceof byte[] bytes) { + return new String(bytes, StandardCharsets.UTF_8); + } + if (value instanceof Clob clob) { + return readClob(clob); + } + if (value instanceof Blob blob) { + return blob.getBytes(1, Math.toIntExact(blob.length())); + } + if (value instanceof SQLXML xml) { + return xml.getString(); + } + return value; + } + + void bind( + PreparedStatement statement, + int index, + Object value, + String table, + MetadataTableDescriptor.Column targetColumn, + MetadataDatabaseKind targetKind) throws SQLException { + if (isPostgresqlOidText(targetKind, table, targetColumn.name())) { + if (value == null) { + statement.setNull(index, Types.BINARY); + } else { + statement.setBytes(index, value.toString().getBytes(StandardCharsets.UTF_8)); + } + } else if (value == null) { + statement.setNull(index, targetColumn.jdbcType()); + } else if (value instanceof byte[] bytes) { + statement.setBytes(index, bytes); + } else if (value instanceof OffsetDateTime timestamp) { + bindOffsetTimestamp(statement, index, timestamp, targetColumn, targetKind); + } else if (value instanceof ZonedDateTime timestamp) { + bindOffsetTimestamp(statement, index, timestamp.toOffsetDateTime(), targetColumn, targetKind); + } else { + statement.setObject(index, value); + } + } + + String selectExpression( + String quotedColumn, + String table, + MetadataTableDescriptor.Column column, + MetadataDatabaseKind kind) { + return isPostgresqlOidText(kind, table, column.name()) + ? "lo_get(" + quotedColumn + ")" + : quotedColumn; + } + + String insertExpression( + String table, + MetadataTableDescriptor.Column column, + MetadataDatabaseKind kind) { + return isPostgresqlOidText(kind, table, column.name()) ? "lo_from_bytea(0, ?)" : "?"; + } + + private static boolean isPostgresqlOidText( + MetadataDatabaseKind kind, + String table, + String column) { + return kind == MetadataDatabaseKind.POSTGRESQL && POSTGRESQL_OID_TEXT.contains(table + '.' + column); + } + + private static void bindOffsetTimestamp( + PreparedStatement statement, + int index, + OffsetDateTime value, + MetadataTableDescriptor.Column targetColumn, + MetadataDatabaseKind targetKind) throws SQLException { + OffsetDateTime utc = value.withOffsetSameInstant(ZoneOffset.UTC); + if (targetKind == MetadataDatabaseKind.MYSQL || !targetColumn.timestampWithTimeZone()) { + statement.setObject(index, utc.toLocalDateTime()); + } else { + statement.setObject(index, utc); + } + } + + private static String readClob(Clob clob) throws SQLException { + try (Reader reader = clob.getCharacterStream()) { + StringBuilder value = new StringBuilder(); + char[] buffer = new char[4096]; + int count; + while ((count = reader.read(buffer)) >= 0) { + value.append(buffer, 0, count); + } + return value.toString(); + } catch (IOException exception) { + throw new SQLException("Cannot materialize large text", "58000"); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationErrorCode.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationErrorCode.java new file mode 100644 index 0000000000..7af17e7ffa --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationErrorCode.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +/** Stable failure classes for the JDBC metadata copy boundary. */ +public enum MetadataMigrationErrorCode { + SCHEMA, + COPY, + VERIFICATION, + SEQUENCE, + TIMEOUT, + ROLLBACK_OUTCOME_UNKNOWN, + COMMIT_OUTCOME_UNKNOWN +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationException.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationException.java new file mode 100644 index 0000000000..c30edd7ad8 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationException.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; + +/** Redacted metadata migration failure. Driver diagnostics deliberately remain outside this boundary. */ +public final class MetadataMigrationException extends RuntimeException { + + private final MetadataMigrationErrorCode code; + + MetadataMigrationException(MetadataMigrationErrorCode code) { + super("Metadata migration failed: " + Objects.requireNonNull(code, "code").name().toLowerCase()); + this.code = code; + } + + public MetadataMigrationErrorCode code() { + return code; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationProgressSink.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationProgressSink.java new file mode 100644 index 0000000000..6f02cfb315 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationProgressSink.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +/** Receives only a coarse stage and total percentage; data details are intentionally unrepresentable. */ +@FunctionalInterface +public interface MetadataMigrationProgressSink { + + MetadataMigrationProgressSink NO_OP = (stage, percent) -> { }; + + void report(MetadataMigrationStage stage, int percent); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationSession.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationSession.java new file mode 100644 index 0000000000..1d2eb14c42 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationSession.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Locale; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Owns transactions on caller-owned JDBC connections without owning connection lifetime. */ +final class MetadataMigrationSession implements AutoCloseable { + + private final Connection source; + private final Connection target; + private final boolean sourceReadOnly; + private final int sourceIsolation; + private final int targetIsolation; + private boolean sourceIsolationChanged; + private boolean sourceReadOnlyChanged; + private boolean sourceAutoCommitChanged; + private boolean targetAutoCommitChanged; + private boolean targetIsolationChanged; + private boolean sourceMutationUncertain; + private boolean targetMutationUncertain; + private boolean commitAttempted; + private boolean committed; + private boolean sourceInvalidated; + private boolean targetInvalidated; + + MetadataMigrationSession(Connection source, Connection target) throws SQLException { + this.source = source; + this.target = target; + sourceReadOnly = source.isReadOnly(); + sourceIsolation = source.getTransactionIsolation(); + targetIsolation = target.getTransactionIsolation(); + } + + void begin(MetadataDatabaseKind targetKind) throws SQLException { + if (!source.getAutoCommit() || !target.getAutoCommit()) { + throw new SQLException("Migration connections must not have active transactions", "25001"); + } + requireDatabase(source, MetadataDatabaseKind.H2); + requireDatabase(target, targetKind); + try { + sourceMutationUncertain = true; + source.setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE); + sourceIsolationChanged = true; + sourceMutationUncertain = false; + sourceMutationUncertain = true; + source.setReadOnly(true); + sourceReadOnlyChanged = true; + sourceMutationUncertain = false; + sourceMutationUncertain = true; + source.setAutoCommit(false); + sourceAutoCommitChanged = true; + sourceMutationUncertain = false; + targetMutationUncertain = true; + target.setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE); + targetIsolationChanged = true; + targetMutationUncertain = false; + targetMutationUncertain = true; + target.setAutoCommit(false); + targetAutoCommitChanged = true; + targetMutationUncertain = false; + } catch (SQLException exception) { + closeUncertainConnections(); + throw exception; + } + } + + void commit() throws SQLException { + commitAttempted = true; + try { + target.commit(); + committed = true; + } catch (SQLException exception) { + invalidateTarget(); + throw new MetadataMigrationException(MetadataMigrationErrorCode.COMMIT_OUTCOME_UNKNOWN); + } + } + + @Override + public void close() { + boolean failed = false; + boolean rollbackOutcomeUnknown = false; + if (commitAttempted && !committed) { + invalidateTarget(); + } else if (targetMutationUncertain) { + failed = true; + invalidateTarget(); + } else if (targetAutoCommitChanged) { + if (!committed && !rollback(target)) { + failed = true; + rollbackOutcomeUnknown = true; + invalidateTarget(); + } else if (!restoreTarget()) { + failed = true; + invalidateTarget(); + } + } + if (sourceMutationUncertain) { + failed = true; + invalidateSource(); + } else if (!restoreSource()) { + failed = true; + invalidateSource(); + } + if (failed && !committed) { + throw new MetadataMigrationException(rollbackOutcomeUnknown + ? MetadataMigrationErrorCode.ROLLBACK_OUTCOME_UNKNOWN + : MetadataMigrationErrorCode.COPY); + } + } + + private static void requireDatabase(Connection connection, MetadataDatabaseKind kind) throws SQLException { + String product = connection.getMetaData().getDatabaseProductName().toLowerCase(Locale.ROOT); + boolean matches = switch (kind) { + case H2 -> product.equals("h2"); + case MYSQL -> product.contains("mysql"); + case POSTGRESQL -> product.contains("postgresql"); + }; + if (!matches) { + throw new SQLException("Unexpected database kind", "55000"); + } + } + + private boolean restoreSource() { + if (sourceAutoCommitChanged && !rollback(source)) { + return false; + } + if (sourceAutoCommitChanged && !restoreAutoCommit(source)) { + return false; + } + try { + if (sourceReadOnlyChanged) { + source.setReadOnly(sourceReadOnly); + } + if (sourceIsolationChanged) { + source.setTransactionIsolation(sourceIsolation); + } + return true; + } catch (SQLException ignored) { + return false; + } + } + + private boolean restoreTarget() { + if (!restoreAutoCommit(target)) { + return false; + } + try { + if (targetIsolationChanged) { + target.setTransactionIsolation(targetIsolation); + } + return true; + } catch (SQLException ignored) { + return false; + } + } + + private void closeUncertainConnections() { + if (targetMutationUncertain) { + invalidateTarget(); + } + if (sourceMutationUncertain) { + invalidateSource(); + } + } + + private void invalidateSource() { + if (!sourceInvalidated) { + sourceInvalidated = true; + closeQuietly(source); + } + } + + private void invalidateTarget() { + if (!targetInvalidated) { + targetInvalidated = true; + closeQuietly(target); + } + } + + private static boolean restoreAutoCommit(Connection connection) { + try { + connection.setAutoCommit(true); + return true; + } catch (SQLException ignored) { + return false; + } + } + + private static boolean rollback(Connection connection) { + try { + connection.rollback(); + return true; + } catch (SQLException ignored) { + return false; + } + } + + private static void closeQuietly(Connection connection) { + try { + connection.close(); + } catch (SQLException ignored) { + // Connection invalidation is best effort; no driver detail may escape. + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationStage.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationStage.java new file mode 100644 index 0000000000..0083285e04 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationStage.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +/** Coarse, safe progress stages. */ +public enum MetadataMigrationStage { + INSPECTING, + COPYING, + VERIFYING, + REPAIRING, + COMPLETE +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataRowCopier.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataRowCopier.java new file mode 100644 index 0000000000..628b64544d --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataRowCopier.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.StringJoiner; +import java.util.stream.Collectors; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Copies one table with explicit columns and primary-key-stable source reads. */ +final class MetadataRowCopier { + + private static final int BATCH_SIZE = 128; + private final MetadataJdbcValueAdapter values; + + MetadataRowCopier(MetadataJdbcValueAdapter values) { + this.values = values; + } + + void copy( + Connection source, + Connection target, + MetadataTableDescriptor sourceTable, + MetadataTableDescriptor targetTable, + MetadataDatabaseKind targetKind, + MigrationDeadline deadline) throws SQLException { + String sourceSql = sourceSelect(sourceTable); + String targetSql = targetInsert(sourceTable, targetTable, targetKind); + try (PreparedStatement reader = source.prepareStatement(sourceSql); + PreparedStatement writer = target.prepareStatement(targetSql)) { + deadline.apply(reader); + deadline.apply(writer); + reader.setFetchSize(BATCH_SIZE); + try (ResultSet rows = reader.executeQuery()) { + int pending = 0; + while (rows.next()) { + deadline.check(); + for (int index = 0; index < sourceTable.columns().size(); index++) { + MetadataTableDescriptor.Column sourceColumn = sourceTable.columns().get(index); + Object value = values.read( + rows, index + 1, sourceTable.name(), sourceColumn, MetadataDatabaseKind.H2); + values.bind( + writer, + index + 1, + value, + targetTable.name(), + targetTable.column(sourceColumn.name()), + targetKind); + } + writer.addBatch(); + pending++; + if (pending == BATCH_SIZE) { + writer.executeBatch(); + pending = 0; + } + } + if (pending > 0) { + writer.executeBatch(); + } + } + } + } + + private static String sourceSelect(MetadataTableDescriptor table) { + String columns = table.columns().stream() + .map(MetadataTableDescriptor.Column::name) + .map(column -> CanonicalTableDigest.quote(column, MetadataDatabaseKind.H2)) + .collect(Collectors.joining(", ")); + String order = table.primaryKey().stream() + .map(column -> CanonicalTableDigest.quote(column, MetadataDatabaseKind.H2)) + .collect(Collectors.joining(", ")); + return "SELECT " + columns + " FROM " + + CanonicalTableDigest.quote(table.name(), MetadataDatabaseKind.H2) + " ORDER BY " + order; + } + + private String targetInsert( + MetadataTableDescriptor sourceTable, + MetadataTableDescriptor targetTable, + MetadataDatabaseKind kind) { + String columns = sourceTable.columns().stream() + .map(MetadataTableDescriptor.Column::name) + .map(column -> CanonicalTableDigest.quote(column, kind)) + .collect(Collectors.joining(", ")); + StringJoiner expressions = new StringJoiner(", "); + sourceTable.columns().stream() + .map(column -> targetTable.column(column.name())) + .forEach(column -> expressions.add(values.insertExpression(targetTable.name(), column, kind))); + return "INSERT INTO " + CanonicalTableDigest.quote(targetTable.name(), kind) + + " (" + columns + ") VALUES (" + expressions + ')'; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataSchemaInventory.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataSchemaInventory.java new file mode 100644 index 0000000000..e845e55e27 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataSchemaInventory.java @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.PriorityQueue; +import java.util.Set; +import java.util.TreeMap; +import java.util.stream.Collectors; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Reads the exact application-table, column, primary-key, and foreign-key inventory. */ +final class MetadataSchemaInventory { + + private static final Set HOUSEKEEPING_TABLES = + Set.of("flyway_schema_history", TargetSchemaContract.TABLE); + private static final String[] OBJECT_TYPES = {"TABLE", "VIEW", "MATERIALIZED VIEW"}; + + private final Map tables; + private final MetadataDatabaseKind kind; + + private MetadataSchemaInventory( + Map tables, + MetadataDatabaseKind kind) { + this.tables = Map.copyOf(tables); + this.kind = kind; + } + + static MetadataSchemaInventory capture( + Connection connection, + Set expectedTables, + MetadataDatabaseKind kind, + MigrationDeadline deadline) throws SQLException { + DatabaseMetaData metadata = connection.getMetaData(); + String catalog = connection.getCatalog(); + String schema = connection.getSchema(); + Set normalizedExpected = expectedTables.stream() + .map(MetadataSchemaInventory::normalize) + .collect(Collectors.toUnmodifiableSet()); + Set actualObjects = readObjects(metadata, catalog, schema); + if (!actualObjects.containsAll(normalizedExpected) + || !normalizedExpected.containsAll(withoutHousekeeping(actualObjects))) { + throw new SQLException("Application schema inventory differs", "55000"); + } + Map descriptors = new TreeMap<>(); + for (String table : normalizedExpected.stream().sorted().toList()) { + deadline.check(); + descriptors.put(table, descriptor(metadata, catalog, schema, table)); + } + return new MetadataSchemaInventory(descriptors, kind); + } + + List foreignKeyOrder() throws SQLException { + Map incoming = new HashMap<>(); + Map> children = new HashMap<>(); + tables.keySet().forEach(table -> incoming.put(table, 0)); + for (MetadataTableDescriptor table : tables.values()) { + for (MetadataTableDescriptor.ForeignKey key : table.foreignKeys()) { + if (key.referencedTable().equals(table.name())) { + continue; + } + if (!tables.containsKey(key.referencedTable())) { + throw new SQLException("Foreign key references another schema", "55000"); + } + incoming.compute(table.name(), (ignored, count) -> count + 1); + children.computeIfAbsent(key.referencedTable(), ignored -> new ArrayList<>()).add(table.name()); + } + } + PriorityQueue ready = new PriorityQueue<>(); + incoming.forEach((table, count) -> { + if (count == 0) { + ready.add(table); + } + }); + List ordered = new ArrayList<>(); + while (!ready.isEmpty()) { + String parent = ready.remove(); + ordered.add(tables.get(parent)); + for (String child : children.getOrDefault(parent, List.of()).stream().sorted().toList()) { + int count = incoming.compute(child, (ignored, current) -> current - 1); + if (count == 0) { + ready.add(child); + } + } + } + if (ordered.size() != tables.size()) { + throw new SQLException("Application foreign keys contain a cycle", "55000"); + } + return List.copyOf(ordered); + } + + boolean hasSamePortableShape(MetadataSchemaInventory other) { + if (!tables.keySet().equals(other.tables.keySet())) { + return false; + } + return tables.entrySet().stream() + .allMatch(entry -> entry.getValue() + .hasSamePortableShape(other.tables.get(entry.getKey()), kind, other.kind)); + } + + MetadataTableDescriptor table(String name) { + return tables.get(name); + } + + private static Set readObjects(DatabaseMetaData metadata, String catalog, String schema) + throws SQLException { + HashSet objects = new HashSet<>(); + try (ResultSet rows = metadata.getTables(catalog, schema, "%", OBJECT_TYPES)) { + while (rows.next()) { + objects.add(normalize(rows.getString("TABLE_NAME"))); + } + } + return Set.copyOf(objects); + } + + private static Set withoutHousekeeping(Set names) { + HashSet application = new HashSet<>(names); + application.removeAll(HOUSEKEEPING_TABLES); + return Set.copyOf(application); + } + + private static MetadataTableDescriptor descriptor( + DatabaseMetaData metadata, + String catalog, + String schema, + String table) throws SQLException { + List columns = readColumns(metadata, catalog, schema, table); + List primaryKey = readPrimaryKey(metadata, catalog, schema, table); + List foreignKeys = readForeignKeys(metadata, catalog, schema, table); + if (columns.isEmpty() || primaryKey.isEmpty()) { + throw new SQLException("Application table is missing columns or primary key", "55000"); + } + return new MetadataTableDescriptor(table, columns, primaryKey, foreignKeys); + } + + private static List readColumns( + DatabaseMetaData metadata, + String catalog, + String schema, + String table) throws SQLException { + Map columns = new TreeMap<>(); + try (ResultSet rows = metadata.getColumns(catalog, schema, table, null)) { + while (rows.next()) { + String autoIncrement = rows.getString("IS_AUTOINCREMENT"); + columns.put(rows.getInt("ORDINAL_POSITION"), new MetadataTableDescriptor.Column( + rows.getString("COLUMN_NAME"), + rows.getInt("DATA_TYPE"), + rows.getString("TYPE_NAME"), + rows.getInt("COLUMN_SIZE"), + rows.getInt("DECIMAL_DIGITS"), + rows.getInt("NULLABLE") == DatabaseMetaData.columnNullable, + "YES".equalsIgnoreCase(autoIncrement))); + } + } + return List.copyOf(columns.values()); + } + + private static List readPrimaryKey( + DatabaseMetaData metadata, + String catalog, + String schema, + String table) throws SQLException { + Map columns = new TreeMap<>(); + try (ResultSet rows = metadata.getPrimaryKeys(catalog, schema, table)) { + while (rows.next()) { + columns.put(rows.getShort("KEY_SEQ"), normalize(rows.getString("COLUMN_NAME"))); + } + } + return List.copyOf(columns.values()); + } + + private static List readForeignKeys( + DatabaseMetaData metadata, + String catalog, + String schema, + String table) throws SQLException { + Map keys = new LinkedHashMap<>(); + int unnamed = 0; + try (ResultSet rows = metadata.getImportedKeys(catalog, schema, table)) { + while (rows.next()) { + String keyName = rows.getString("FK_NAME"); + if (keyName == null && rows.getShort("KEY_SEQ") == 1) { + unnamed++; + } + String group = keyName == null ? "unnamed-" + unnamed : normalize(keyName); + String referencedTable = normalize(rows.getString("PKTABLE_NAME")); + keys.computeIfAbsent(group, ignored -> new ForeignKeyBuilder(referencedTable)) + .add( + rows.getShort("KEY_SEQ"), + normalize(rows.getString("FKCOLUMN_NAME")), + normalize(rows.getString("PKCOLUMN_NAME"))); + } + } + return keys.values().stream() + .map(ForeignKeyBuilder::build) + .sorted(Comparator.comparing(MetadataTableDescriptor.ForeignKey::referencedTable) + .thenComparing(key -> String.join(",", key.columns()))) + .toList(); + } + + private static String normalize(String value) { + return value.toLowerCase(Locale.ROOT); + } + + private static final class ForeignKeyBuilder { + + private final String referencedTable; + private final Map columns = new TreeMap<>(); + private final Map referencedColumns = new TreeMap<>(); + + private ForeignKeyBuilder(String referencedTable) { + this.referencedTable = referencedTable; + } + + void add(short position, String column, String referencedColumn) { + columns.put(position, column); + referencedColumns.put(position, referencedColumn); + } + + MetadataTableDescriptor.ForeignKey build() { + return new MetadataTableDescriptor.ForeignKey( + List.copyOf(columns.values()), referencedTable, List.copyOf(referencedColumns.values())); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataTableDescriptor.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataTableDescriptor.java new file mode 100644 index 0000000000..3dabf69331 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataTableDescriptor.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Types; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Set; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Immutable JDBC schema descriptor for one application table. */ +record MetadataTableDescriptor( + String name, + List columns, + List primaryKey, + List foreignKeys) { + + MetadataTableDescriptor { + name = normalized(name); + columns = List.copyOf(columns); + primaryKey = primaryKey.stream().map(MetadataTableDescriptor::normalized).toList(); + foreignKeys = List.copyOf(foreignKeys); + } + + boolean hasSamePortableShape( + MetadataTableDescriptor other, + MetadataDatabaseKind kind, + MetadataDatabaseKind otherKind) { + return name.equals(other.name) + && columns.size() == other.columns.size() + && columns.stream().allMatch(column -> column.hasSameSemantics( + other.column(column.name()), kind, otherKind, name)) + && primaryKey.equals(other.primaryKey) + && foreignKeys.equals(other.foreignKeys); + } + + Column column(String columnName) { + return columns.stream() + .filter(column -> column.name().equals(columnName)) + .findFirst() + .orElse(null); + } + + List identityColumns() { + return columns.stream().filter(Column::autoIncrement).toList(); + } + + private static String normalized(String value) { + return Objects.requireNonNull(value, "identifier").toLowerCase(Locale.ROOT); + } + + record Column( + String name, + int jdbcType, + String typeName, + int size, + int scale, + boolean nullable, + boolean autoIncrement) { + + Column { + name = normalized(name); + typeName = Objects.requireNonNullElse(typeName, "").toLowerCase(Locale.ROOT); + } + + boolean hasSameSemantics( + Column other, + MetadataDatabaseKind kind, + MetadataDatabaseKind otherKind, + String table) { + if (other == null || !name.equals(other.name) || autoIncrement != other.autoIncrement) { + return false; + } + String type = semanticType(kind, table); + String otherType = other.semanticType(otherKind, table); + return other != null + && (type.equals(otherType) + || booleanIntegerCompatibility(type, otherType) + || textCompatibility(type, otherType)); + } + + private static boolean booleanIntegerCompatibility(String type, String otherType) { + return Set.of(type, otherType).equals(Set.of("boolean", "small-integer")); + } + + private static boolean textCompatibility(String type, String otherType) { + return Set.of(type, otherType).equals(Set.of("character", "large-text")); + } + + String semanticType( + MetadataDatabaseKind kind, + String table) { + if (kind == MetadataDatabaseKind.POSTGRESQL + && typeName.equals("oid") && name.equals("content") + && Set.of("hzb_ai_message", "hzb_define", "hzb_notice_template").contains(table)) { + return "large-text"; + } + if ((jdbcType == Types.VARCHAR || jdbcType == Types.NVARCHAR) && size >= 1_000_000) { + return "large-text"; + } + if (typeName.startsWith("enum")) { + return "character"; + } + return switch (jdbcType) { + case Types.BOOLEAN -> "boolean"; + case Types.BIT -> size == 1 ? "boolean" : "binary-bit(" + size + ')'; + case Types.TINYINT -> kind == MetadataDatabaseKind.MYSQL + && size == 1 ? "boolean" : "small-integer"; + case Types.SMALLINT -> "small-integer"; + case Types.INTEGER -> "integer"; + case Types.BIGINT -> "bigint"; + case Types.NUMERIC, Types.DECIMAL -> "decimal(" + size + ',' + scale + ')'; + case Types.REAL, Types.FLOAT, Types.DOUBLE -> "floating"; + case Types.CHAR, Types.NCHAR, Types.VARCHAR, Types.NVARCHAR -> "character"; + case Types.LONGVARCHAR, Types.LONGNVARCHAR, Types.CLOB, Types.NCLOB -> "large-text"; + case Types.BINARY, Types.VARBINARY -> "binary"; + case Types.LONGVARBINARY, Types.BLOB -> "large-binary"; + case Types.DATE -> "date"; + case Types.TIME, Types.TIME_WITH_TIMEZONE -> "time"; + case Types.TIMESTAMP, Types.TIMESTAMP_WITH_TIMEZONE -> "timestamp"; + default -> "jdbc-type(" + jdbcType + ')'; + }; + } + + boolean timestampWithTimeZone() { + return jdbcType == Types.TIMESTAMP_WITH_TIMEZONE + || typeName.equals("timestamptz") + || typeName.contains("timestamp with time zone"); + } + } + + record ForeignKey(List columns, String referencedTable, List referencedColumns) { + + ForeignKey { + columns = columns.stream().map(MetadataTableDescriptor::normalized).toList(); + referencedTable = normalized(referencedTable); + referencedColumns = referencedColumns.stream().map(MetadataTableDescriptor::normalized).toList(); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationDeadline.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationDeadline.java new file mode 100644 index 0000000000..53c5dd472e --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationDeadline.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +/** Monotonic budget used for query timeouts and cooperative statement-loop cancellation. */ +final class MigrationDeadline { + + private final long deadlineNanos; + + MigrationDeadline(Duration timeout) { + Objects.requireNonNull(timeout, "timeout"); + if (timeout.isZero() || timeout.isNegative()) { + throw new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT); + } + long durationNanos; + try { + durationNanos = timeout.toNanos(); + } catch (ArithmeticException exception) { + durationNanos = Long.MAX_VALUE; + } + long now = System.nanoTime(); + deadlineNanos = durationNanos > Long.MAX_VALUE - now ? Long.MAX_VALUE : now + durationNanos; + } + + void apply(Statement statement) throws SQLException { + statement.setQueryTimeout(remainingSeconds()); + } + + void check() { + if (Thread.currentThread().isInterrupted() || deadlineNanos - System.nanoTime() <= 0) { + throw new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT); + } + } + + int remainingSeconds() { + long remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0) { + throw new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT); + } + long seconds = Math.max(1, TimeUnit.NANOSECONDS.toSeconds(remaining)); + return (int) Math.min(Integer.MAX_VALUE, seconds); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaContract.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaContract.java index e5514ff611..f9fec6166d 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaContract.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaContract.java @@ -62,13 +62,22 @@ final class TargetSchemaContract { } boolean matches(Connection connection, Set baselineTables) throws SQLException { - return JdbcTargetSchemaState.capture(connection, kind, baselineTables).equals(readRecordedState(connection)); + return matches(connection, baselineTables, 0); } - private JdbcTargetSchemaState.SchemaState readRecordedState(Connection connection) throws SQLException { + boolean matches(Connection connection, Set baselineTables, int queryTimeoutSeconds) throws SQLException { + return JdbcTargetSchemaState.capture(connection, kind, baselineTables, queryTimeoutSeconds) + .equals(readRecordedState(connection, queryTimeoutSeconds)); + } + + private JdbcTargetSchemaState.SchemaState readRecordedState( + Connection connection, int queryTimeoutSeconds) throws SQLException { Map facts = new TreeMap<>(); String select = "SELECT database_kind, definition, occurrences FROM " + TABLE; try (PreparedStatement statement = connection.prepareStatement(select)) { + if (queryTimeoutSeconds > 0) { + statement.setQueryTimeout(queryTimeoutSeconds); + } try (ResultSet rows = statement.executeQuery()) { while (rows.next()) { if (!kind.name().equals(rows.getString("database_kind"))) { diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationContractTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationContractTest.java new file mode 100644 index 0000000000..d4843d8a22 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationContractTest.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.DriverManager; +import java.sql.SQLTimeoutException; +import java.sql.Statement; +import java.sql.Types; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.api.Test; + +class JdbcMetadataMigrationContractTest { + + @Test + void renamedColumnIsAnIncompatibleShapeInsteadOfAnInspectionFailure() { + MetadataTableDescriptor source = descriptor("name"); + MetadataTableDescriptor target = descriptor("renamed"); + + assertThat(source.hasSamePortableShape( + target, MetadataDatabaseKind.H2, MetadataDatabaseKind.MYSQL)).isFalse(); + } + + @Test + void rejectsUnexpectedSchemaWithoutClosingCallerConnectionsOrLeakingDetails() throws Exception { + String sourceUrl = "jdbc:h2:mem:copy-contract-source;DB_CLOSE_DELAY=-1"; + String targetUrl = "jdbc:h2:mem:copy-contract-target;DB_CLOSE_DELAY=-1"; + List progress = new ArrayList<>(); + try (Connection source = DriverManager.getConnection(sourceUrl); + Connection target = DriverManager.getConnection(targetUrl); + Statement statement = source.createStatement()) { + statement.execute("CREATE TABLE private_source_table (id BIGINT PRIMARY KEY, secret VARCHAR(64))"); + + assertThatThrownBy(() -> new JdbcMetadataMigration().migrate( + source, + target, + MetadataDatabaseKind.POSTGRESQL, + Duration.ofSeconds(5), + (stage, percent) -> progress.add(new Progress(stage, percent)))) + .isInstanceOfSatisfying(MetadataMigrationException.class, exception -> { + assertThat(exception.code()).isEqualTo(MetadataMigrationErrorCode.SCHEMA); + assertThat(exception).hasNoCause(); + assertThat(exception.getMessage()) + .doesNotContain(sourceUrl, targetUrl, "private_source_table", "secret"); + }); + + assertThat(source.isClosed()).isFalse(); + assertThat(target.isClosed()).isFalse(); + assertThat(progress).allSatisfy(event -> { + assertThat(event.percent()).isBetween(0, 100); + assertThat(event.toString()).doesNotContain("private_source_table", "secret"); + }); + } + } + + @Test + void rejectsExpiredCallerDeadlineBeforeInspectingJdbcMetadata() throws Exception { + try (Connection source = DriverManager.getConnection("jdbc:h2:mem:expired-source"); + Connection target = DriverManager.getConnection("jdbc:h2:mem:expired-target")) { + assertThatThrownBy(() -> new JdbcMetadataMigration().migrate( + source, + target, + MetadataDatabaseKind.MYSQL, + Duration.ZERO, + MetadataMigrationProgressSink.NO_OP)) + .isInstanceOfSatisfying(MetadataMigrationException.class, exception -> { + assertThat(exception.code()).isEqualTo(MetadataMigrationErrorCode.TIMEOUT); + assertThat(exception).hasNoCause(); + }); + } + } + + @Test + void classifiesSqlTimeoutExceptionWithoutSqlStateAsTimeout() throws Exception { + Connection source = connection("H2"); + Connection target = connection("MySQL"); + doThrow(new SQLTimeoutException("private timeout diagnostic")) + .when(target).setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE); + + assertThatThrownBy(() -> new JdbcMetadataMigration().migrate( + source, + target, + MetadataDatabaseKind.MYSQL, + Duration.ofSeconds(5), + MetadataMigrationProgressSink.NO_OP)) + .isInstanceOfSatisfying(MetadataMigrationException.class, exception -> { + assertThat(exception.code()).isEqualTo(MetadataMigrationErrorCode.TIMEOUT); + assertThat(exception).hasNoCause(); + assertThat(exception.getMessage()).doesNotContain("private timeout diagnostic"); + }); + } + + private record Progress(MetadataMigrationStage stage, int percent) { + } + + private static MetadataTableDescriptor descriptor(String column) { + return new MetadataTableDescriptor( + "sample", + List.of(new MetadataTableDescriptor.Column( + column, Types.BIGINT, "bigint", 64, 0, false, false)), + List.of(column), + List.of()); + } + + private static Connection connection(String product) throws Exception { + Connection connection = mock(Connection.class); + DatabaseMetaData metadata = mock(DatabaseMetaData.class); + when(connection.getMetaData()).thenReturn(metadata); + when(metadata.getDatabaseProductName()).thenReturn(product); + when(connection.getAutoCommit()).thenReturn(true); + when(connection.getTransactionIsolation()).thenReturn(Connection.TRANSACTION_READ_COMMITTED); + when(connection.isReadOnly()).thenReturn(false); + return connection; + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationSessionTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationSessionTest.java new file mode 100644 index 0000000000..d3b9b5282b --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationSessionTest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.api.Test; +import org.mockito.InOrder; + +class MetadataMigrationSessionTest { + + @Test + void rollbackFailureInvalidatesTargetWithoutEnablingAutoCommit() throws Exception { + Connection source = connection("H2"); + Connection target = connection("PostgreSQL"); + doThrow(new SQLException("private rollback diagnostic")).when(target).rollback(); + MetadataMigrationSession session = new MetadataMigrationSession(source, target); + session.begin(MetadataDatabaseKind.POSTGRESQL); + + assertThatThrownBy(session::close) + .isInstanceOfSatisfying(MetadataMigrationException.class, exception -> { + assertThat(exception.code()) + .isEqualTo(MetadataMigrationErrorCode.ROLLBACK_OUTCOME_UNKNOWN); + assertThat(exception).hasNoCause(); + }); + + verify(target).close(); + verify(target, never()).setAutoCommit(true); + } + + @Test + void partialBeginRestoresSourceWhenTargetTransactionCannotStart() throws Exception { + Connection source = connection("H2"); + Connection target = connection("MySQL"); + doThrow(new SQLException("private target diagnostic")).when(target).setAutoCommit(false); + MetadataMigrationSession session = new MetadataMigrationSession(source, target); + + assertThatThrownBy(() -> session.begin(MetadataDatabaseKind.MYSQL)).isInstanceOf(SQLException.class); + assertThatThrownBy(session::close).isInstanceOf(MetadataMigrationException.class).hasNoCause(); + + InOrder sourceOrder = inOrder(source); + sourceOrder.verify(source).setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE); + sourceOrder.verify(source).setReadOnly(true); + sourceOrder.verify(source).setAutoCommit(false); + sourceOrder.verify(source).rollback(); + sourceOrder.verify(source).setAutoCommit(true); + sourceOrder.verify(source).setReadOnly(false); + sourceOrder.verify(source).setTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED); + verify(target).close(); + } + + @Test + void committedRowsRemainSuccessWhenCallerConnectionCannotBeRestored() throws Exception { + Connection source = connection("H2"); + Connection target = connection("MySQL"); + doThrow(new SQLException("private cleanup diagnostic")).when(target).setAutoCommit(true); + MetadataMigrationSession session = new MetadataMigrationSession(source, target); + session.begin(MetadataDatabaseKind.MYSQL); + session.commit(); + + session.close(); + + verify(target).commit(); + verify(target).close(); + verify(target, never()).rollback(); + } + + @Test + void commitExceptionHasAnExplicitOutcomeUnknownCodeAndNeverRollsBackBlindly() throws Exception { + Connection source = connection("H2"); + Connection target = connection("PostgreSQL"); + doThrow(new SQLException("private commit diagnostic")).when(target).commit(); + MetadataMigrationSession session = new MetadataMigrationSession(source, target); + session.begin(MetadataDatabaseKind.POSTGRESQL); + + assertThatThrownBy(session::commit) + .isInstanceOfSatisfying(MetadataMigrationException.class, exception -> { + assertThat(exception.code()).isEqualTo(MetadataMigrationErrorCode.COMMIT_OUTCOME_UNKNOWN); + assertThat(exception).hasNoCause(); + }); + session.close(); + + verify(target, never()).rollback(); + verify(target, never()).setAutoCommit(true); + verify(target).close(); + } + + @Test + void rollbackOutcomeUnknownTakesPriorityOverBodyFailure() throws Exception { + Connection source = connection("H2"); + Connection target = connection("MySQL"); + doThrow(new SQLException("private rollback diagnostic")).when(target).rollback(); + Throwable failure = catchThrowable(() -> { + try (MetadataMigrationSession session = new MetadataMigrationSession(source, target)) { + session.begin(MetadataDatabaseKind.MYSQL); + throw new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT); + } + }); + + assertThat(failure).isInstanceOf(MetadataMigrationException.class); + assertThat(JdbcMetadataMigration.cleanupFailure((Exception) failure).code()) + .isEqualTo(MetadataMigrationErrorCode.ROLLBACK_OUTCOME_UNKNOWN); + } + + private static Connection connection(String product) throws Exception { + Connection connection = mock(Connection.class); + DatabaseMetaData metadata = mock(DatabaseMetaData.class); + when(connection.getMetaData()).thenReturn(metadata); + when(metadata.getDatabaseProductName()).thenReturn(product); + when(connection.getAutoCommit()).thenReturn(true); + when(connection.getTransactionIsolation()).thenReturn(Connection.TRANSACTION_READ_COMMITTED); + when(connection.isReadOnly()).thenReturn(false); + return connection; + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationDatabaseTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationDatabaseTest.java new file mode 100644 index 0000000000..b2ed650f64 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationDatabaseTest.java @@ -0,0 +1,353 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import jakarta.persistence.Entity; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.hibernate.SessionFactory; +import org.hibernate.boot.MetadataSources; +import org.hibernate.boot.registry.StandardServiceRegistry; +import org.hibernate.boot.registry.StandardServiceRegistryBuilder; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.springframework.context.annotation.ClassPathScanningCandidateComponentProvider; +import org.springframework.core.type.filter.AnnotationTypeFilter; +import org.testcontainers.mysql.MySQLContainer; +import org.testcontainers.postgresql.PostgreSQLContainer; + +/** Real B206 copy proof against the supported external metadata databases. */ +@EnabledIfSystemProperty(named = "hertzbeat.test.database-containers", matches = "true") +class JdbcMetadataMigrationDatabaseTest { + + private static final String DATABASE = "hertzbeat"; + private static final String USERNAME = "hertzbeat"; + private static final String PASSWORD = "test-only-password"; + + @Test + void copiesAndVerifiesB206MetadataIntoMysql() throws Exception { + try (MySQLContainer database = new MySQLContainer("mysql:8.4") + .withDatabaseName(DATABASE) + .withUsername(USERNAME) + .withPassword(PASSWORD) + .withCommand("--lower-case-table-names=1", "--log-bin-trust-function-creators=1")) { + database.start(); + proveCopy(database.getJdbcUrl(), MetadataDatabaseKind.MYSQL); + } + } + + @Test + void copiesAndVerifiesB206MetadataAndOidTextIntoPostgresql() throws Exception { + try (PostgreSQLContainer database = new PostgreSQLContainer("postgres:17.6") + .withDatabaseName(DATABASE) + .withUsername(USERNAME) + .withPassword(PASSWORD)) { + database.start(); + proveCopy(database.getJdbcUrl(), MetadataDatabaseKind.POSTGRESQL); + } + } + + private static void proveCopy(String targetUrl, MetadataDatabaseKind targetKind) throws Exception { + String sourceUrl = "jdbc:h2:mem:migration_" + targetKind.value() + + ";MODE=MYSQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE"; + createSourceSchema(sourceUrl); + new FlywayTargetSchemaProvisioner().provision( + new MetadataDatabaseConfiguration(targetKind, targetUrl, USERNAME, PASSWORD)); + try (Connection source = DriverManager.getConnection(sourceUrl, "sa", ""); + Connection target = DriverManager.getConnection(targetUrl, USERNAME, PASSWORD)) { + insertSourceFixtures(source); + assertTargetTriggerRejected(source, target, targetKind); + if (targetKind == MetadataDatabaseKind.POSTGRESQL) { + assertOidRollbackLeavesNoOrphan(source, target, targetKind); + } + List progress = new ArrayList<>(); + assertConcurrentWriterExcluded(source, target, targetUrl, targetKind, progress); + + assertThat(source.isClosed()).isFalse(); + assertThat(target.isClosed()).isFalse(); + assertThat(progress).isNotEmpty(); + assertThat(progress.getLast()).isEqualTo(new Progress(MetadataMigrationStage.COMPLETE, 100)); + assertCopiedValues(target, targetKind); + assertNextIdentifier(target, targetKind); + } + } + + private static void assertOidRollbackLeavesNoOrphan( + Connection source, + Connection target, + MetadataDatabaseKind targetKind) throws Exception { + long largeObjectsBefore = count(target, "SELECT count(*) FROM pg_largeobject_metadata"); + assertThat(count(source, "SELECT count(*) FROM hzb_notice_template WHERE id = 43")) + .isEqualTo(1); + JdbcMetadataMigration failingMigration = new JdbcMetadataMigration(table -> { + if (table.equals("hzb_notice_template")) { + throw new SQLException("Injected copy failure"); + } + }); + assertThatThrownBy(() -> failingMigration.migrate( + source, + target, + targetKind, + Duration.ofMinutes(2), + MetadataMigrationProgressSink.NO_OP)) + .isInstanceOfSatisfying(MetadataMigrationException.class, exception -> + assertThat(exception.code()).isEqualTo(MetadataMigrationErrorCode.COPY)); + assertAllBusinessTablesEmpty(target, targetKind); + assertThat(count(target, "SELECT count(*) FROM pg_largeobject_metadata")) + .isEqualTo(largeObjectsBefore); + } + + private static void assertTargetTriggerRejected( + Connection source, + Connection target, + MetadataDatabaseKind targetKind) throws Exception { + try (Statement statement = target.createStatement()) { + if (targetKind == MetadataDatabaseKind.POSTGRESQL) { + statement.execute("CREATE FUNCTION migration_test_trigger() RETURNS trigger LANGUAGE plpgsql AS " + + "'BEGIN RETURN NEW; END'"); + statement.execute("CREATE TRIGGER migration_test_trigger BEFORE INSERT ON hzb_ai_conversation " + + "FOR EACH ROW EXECUTE FUNCTION migration_test_trigger()"); + } else { + statement.execute("CREATE TRIGGER migration_test_trigger BEFORE INSERT ON hzb_ai_conversation " + + "FOR EACH ROW SET NEW.title = NEW.title"); + } + } + try { + assertThatThrownBy(() -> new JdbcMetadataMigration().migrate( + source, + target, + targetKind, + Duration.ofMinutes(2), + MetadataMigrationProgressSink.NO_OP)) + .isInstanceOfSatisfying(MetadataMigrationException.class, exception -> + assertThat(exception.code()).isEqualTo(MetadataMigrationErrorCode.SCHEMA)); + assertAllBusinessTablesEmpty(target, targetKind); + } finally { + try (Statement statement = target.createStatement()) { + if (targetKind == MetadataDatabaseKind.POSTGRESQL) { + statement.execute("DROP TRIGGER migration_test_trigger ON hzb_ai_conversation"); + statement.execute("DROP FUNCTION migration_test_trigger()"); + } else { + statement.execute("DROP TRIGGER migration_test_trigger"); + } + } + } + } + + private static void assertAllBusinessTablesEmpty( + Connection target, + MetadataDatabaseKind targetKind) throws Exception { + for (String table : TargetSchemaBaseline.load(targetKind).expectedTables()) { + String sql = "SELECT count(*) FROM " + CanonicalTableDigest.quote(table, targetKind); + assertThat(count(target, sql)).as(table).isZero(); + } + } + + private static void assertConcurrentWriterExcluded( + Connection source, + Connection target, + String targetUrl, + MetadataDatabaseKind targetKind, + List progress) throws Exception { + CountDownLatch verified = new CountDownLatch(1); + CountDownLatch releaseCommit = new CountDownLatch(1); + AtomicReference migrationFailure = new AtomicReference<>(); + Thread migration = Thread.ofPlatform().name("metadata-migration-proof").start(() -> { + try { + new JdbcMetadataMigration().migrate( + source, + target, + targetKind, + Duration.ofMinutes(2), + (stage, percent) -> { + progress.add(new Progress(stage, percent)); + if (stage == MetadataMigrationStage.REPAIRING && percent == 90) { + verified.countDown(); + await(releaseCommit); + } + }); + } catch (Throwable exception) { + migrationFailure.set(exception); + } + }); + assertThat(verified.await(30, TimeUnit.SECONDS)).isTrue(); + try (Connection contender = DriverManager.getConnection(targetUrl, USERNAME, PASSWORD); + Statement statement = contender.createStatement()) { + statement.setQueryTimeout(3); + if (targetKind == MetadataDatabaseKind.POSTGRESQL) { + statement.execute("SET lock_timeout = '1s'"); + } else { + statement.execute("SET SESSION innodb_lock_wait_timeout = 1"); + } + assertThatThrownBy(() -> statement.executeUpdate( + "INSERT INTO hzb_ai_conversation (id, title) VALUES (99, 'contender')")) + .isInstanceOf(SQLException.class); + } finally { + releaseCommit.countDown(); + } + migration.join(TimeUnit.SECONDS.toMillis(30)); + assertThat(migration.isAlive()).isFalse(); + assertThat(migrationFailure.get()).isNull(); + } + + private static void await(CountDownLatch latch) { + try { + if (!latch.await(30, TimeUnit.SECONDS)) { + throw new IllegalStateException("Timed out waiting for the concurrency proof"); + } + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Concurrency proof was interrupted"); + } + } + + private static long count(Connection connection, String sql) throws Exception { + try (Statement statement = connection.createStatement(); ResultSet rows = statement.executeQuery(sql)) { + assertThat(rows.next()).isTrue(); + return rows.getLong(1); + } + } + + private static void insertSourceFixtures(Connection source) throws Exception { + try (Statement statement = source.createStatement()) { + statement.executeUpdate("INSERT INTO hzb_ai_conversation (id, title, gmt_create) " + + "VALUES (41, 'source', TIMESTAMP '2026-08-09 01:02:03.456789')"); + statement.executeUpdate("INSERT INTO hzb_ai_message " + + "(id, conversation_id, content, role) VALUES " + + "(42, 41, 'Unicode é \u7A7A text', 'user')"); + statement.executeUpdate("INSERT INTO hzb_define (app, content) VALUES ('Z ', 'define-content')"); + statement.executeUpdate("INSERT INTO hzb_define (app, content) VALUES ('a', 'second-content')"); + statement.executeUpdate("INSERT INTO hzb_define (app, content) VALUES ('Ω', 'omega-content')"); + statement.executeUpdate("INSERT INTO hzb_define (app, content) VALUES ('\u4E2D', 'cjk-content')"); + statement.executeUpdate("INSERT INTO hzb_notice_template " + + "(id, name, type, preset, content) VALUES (43, 'template', 1, true, 'notice-content')"); + } + OffsetDateTime start = OffsetDateTime.of( + LocalDateTime.of(2026, 8, 9, 11, 12, 13, 456789000), ZoneOffset.ofHoursMinutes(5, 30)); + try (PreparedStatement statement = source.prepareStatement("INSERT INTO hzb_alert_silence " + + "(id, name, enable, match_all, type, period_start, period_end) VALUES (?, ?, ?, ?, ?, ?, ?)")) { + statement.setLong(1, 44); + statement.setString(2, "silence"); + statement.setBoolean(3, true); + statement.setBoolean(4, false); + statement.setByte(5, (byte) 0); + statement.setObject(6, start); + statement.setObject(7, start.plusHours(1)); + statement.executeUpdate(); + } + } + + private static void assertCopiedValues(Connection target, MetadataDatabaseKind kind) throws Exception { + String contentSql = kind == MetadataDatabaseKind.POSTGRESQL + ? "SELECT convert_from(lo_get(content), 'UTF8') FROM hzb_ai_message WHERE id = 42" + : "SELECT content FROM hzb_ai_message WHERE id = 42"; + try (Statement statement = target.createStatement(); ResultSet rows = statement.executeQuery(contentSql)) { + assertThat(rows.next()).isTrue(); + assertThat(rows.getString(1)).isEqualTo("Unicode é \u7A7A text"); + } + try (Statement statement = target.createStatement(); + ResultSet rows = statement.executeQuery( + "SELECT period_start FROM hzb_alert_silence WHERE id = 44")) { + assertThat(rows.next()).isTrue(); + if (kind == MetadataDatabaseKind.POSTGRESQL) { + assertThat(rows.getObject(1, OffsetDateTime.class).toInstant()) + .isEqualTo(OffsetDateTime.parse("2026-08-09T11:12:13.456789+05:30").toInstant()); + } else { + assertThat(rows.getObject(1, LocalDateTime.class)) + .isEqualTo(LocalDateTime.parse("2026-08-09T05:42:13.456789")); + } + } + } + + private static void assertNextIdentifier(Connection target, MetadataDatabaseKind kind) throws Exception { + String sql = kind == MetadataDatabaseKind.POSTGRESQL + ? "INSERT INTO hzb_ai_conversation (title) VALUES ('next') RETURNING id" + : "INSERT INTO hzb_ai_conversation (title) VALUES ('next')"; + try (PreparedStatement statement = kind == MetadataDatabaseKind.POSTGRESQL + ? target.prepareStatement(sql) + : target.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS)) { + if (kind == MetadataDatabaseKind.POSTGRESQL) { + try (ResultSet rows = statement.executeQuery()) { + assertThat(rows.next()).isTrue(); + assertThat(rows.getLong(1)).isGreaterThan(41); + } + } else { + statement.executeUpdate(); + try (ResultSet rows = statement.getGeneratedKeys()) { + assertThat(rows.next()).isTrue(); + assertThat(rows.getLong(1)).isGreaterThan(41); + } + } + } + } + + private static void createSourceSchema(String jdbcUrl) { + StandardServiceRegistryBuilder builder = new StandardServiceRegistryBuilder() + .applySetting("jakarta.persistence.jdbc.url", jdbcUrl) + .applySetting("jakarta.persistence.jdbc.user", "sa") + .applySetting("jakarta.persistence.jdbc.password", "") + .applySetting("hibernate.dialect", "org.hibernate.dialect.H2Dialect") + .applySetting("hibernate.physical_naming_strategy", + "org.hibernate.boot.model.naming.CamelCaseToUnderscoresNamingStrategy") + .applySetting("hibernate.hbm2ddl.auto", "create"); + StandardServiceRegistry registry = builder.build(); + try { + MetadataSources sources = new MetadataSources(registry); + ClassPathScanningCandidateComponentProvider scanner = + new ClassPathScanningCandidateComponentProvider(false); + scanner.addIncludeFilter(new AnnotationTypeFilter(Entity.class)); + scanner.findCandidateComponents("org.apache.hertzbeat").stream() + .map(definition -> loadClass(definition.getBeanClassName())) + .forEach(sources::addAnnotatedClass); + try (SessionFactory factory = sources.buildMetadata().buildSessionFactory()) { + assertThat(factory.getMetamodel().getEntities()).hasSize(48); + } + } finally { + StandardServiceRegistryBuilder.destroy(registry); + } + } + + private static Class loadClass(String className) { + try { + return Class.forName(className); + } catch (ClassNotFoundException exception) { + throw new IllegalStateException("Mapped entity class is unavailable", exception); + } + } + + private record Progress(MetadataMigrationStage stage, int percent) { + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisionerDatabaseTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisionerDatabaseTest.java index 9e7af00c86..271364de5b 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisionerDatabaseTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisionerDatabaseTest.java @@ -305,6 +305,39 @@ class TargetSchemaProvisionerDatabaseTest { target.kind() == MetadataDatabaseKind.MYSQL ? "ALTER TABLE hzb_account MODIFY COLUMN credential_version BIGINT NOT NULL" : "ALTER TABLE hzb_account ALTER COLUMN credential_version TYPE BIGINT"); + assertCorruptionRejected(provisioner, target, + target.kind() == MetadataDatabaseKind.MYSQL + ? "ALTER TABLE hzb_alert_silence MODIFY COLUMN id BIGINT NOT NULL" + : "ALTER TABLE hzb_alert_silence ALTER COLUMN id DROP IDENTITY", + target.kind() == MetadataDatabaseKind.MYSQL + ? "ALTER TABLE hzb_alert_silence MODIFY COLUMN id BIGINT NOT NULL AUTO_INCREMENT" + : "ALTER TABLE hzb_alert_silence ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY"); + assertCorruptionRejected(provisioner, target, + "ALTER TABLE hzb_auth_token ALTER COLUMN token_scope SET DEFAULT 'API-ADMIN'", + "ALTER TABLE hzb_auth_token ALTER COLUMN token_scope SET DEFAULT 'api-admin'"); + assertCorruptionRejected(provisioner, target, + "ALTER TABLE hzb_auth_token ALTER COLUMN token_scope DROP DEFAULT", + "ALTER TABLE hzb_auth_token ALTER COLUMN token_scope SET DEFAULT 'api-admin'"); + if (target.kind() == MetadataDatabaseKind.POSTGRESQL) { + assertCorruptionRejected(provisioner, target, + "ALTER TABLE hzb_notice_template DROP CONSTRAINT hzb_notice_template_type_check", + "ALTER TABLE hzb_notice_template ADD CONSTRAINT hzb_notice_template_type_check " + + "CHECK (type >= 0)"); + assertCorruptionRejected(provisioner, target, + "CREATE SEQUENCE migration_unexpected_sequence", + "DROP SEQUENCE migration_unexpected_sequence"); + assertCorruptionRejected(provisioner, target, + "ALTER SEQUENCE hzb_auth_token_id_seq INCREMENT BY 2", + "ALTER SEQUENCE hzb_auth_token_id_seq INCREMENT BY 1"); + assertCorruptionRejected(provisioner, target, + "ALTER SEQUENCE hzb_auth_token_id_seq CYCLE", + "ALTER SEQUENCE hzb_auth_token_id_seq NO CYCLE"); + assertCorruptionRejected(provisioner, target, + "ALTER SEQUENCE hzb_auth_token_id_seq OWNED BY hzb_signal_saved_view.id; " + + "ALTER SEQUENCE hzb_signal_saved_view_id_seq OWNED BY hzb_auth_token.id", + "ALTER SEQUENCE hzb_auth_token_id_seq OWNED BY hzb_auth_token.id; " + + "ALTER SEQUENCE hzb_signal_saved_view_id_seq OWNED BY hzb_signal_saved_view.id"); + } assertCorruptionRejected(provisioner, target, target.kind() == MetadataDatabaseKind.MYSQL ? "DROP INDEX idx_hzb_monitor_app ON hzb_monitor" From aa3704bd736a97e4ee669ac9dc05a367bdba62f2 Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 05:57:09 +0800 Subject: [PATCH 41/71] Version metadata migration journal --- .../workflow/MigrationOperationFileCodec.java | 18 ++- .../workflow/MigrationOperationSnapshot.java | 32 ++++- .../MigrationOperationTransitionPolicy.java | 3 + .../FileMigrationOperationStoreTest.java | 126 ++++++++++++++++-- ...igrationOperationTransitionPolicyTest.java | 35 ++++- 5 files changed, 196 insertions(+), 18 deletions(-) diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationFileCodec.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationFileCodec.java index dbe655ac13..585022d2ac 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationFileCodec.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationFileCodec.java @@ -24,11 +24,13 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; final class MigrationOperationFileCodec { private static final String ABSENT = "-"; - private static final int FIELD_COUNT = 16; + private static final String SCHEMA_VERSION = "2"; + private static final int FIELD_COUNT = 18; private final MigrationOperationCollectionPolicy collectionPolicy = new MigrationOperationCollectionPolicy(); byte[] encode(List snapshots) { - StringBuilder output = new StringBuilder("schema=1\ncount=").append(snapshots.size()).append('\n'); + StringBuilder output = new StringBuilder("schema=").append(SCHEMA_VERSION) + .append("\ncount=").append(snapshots.size()).append('\n'); for (int index = 0; index < snapshots.size(); index++) { append(output, index, snapshots.get(index)); } @@ -37,7 +39,7 @@ final class MigrationOperationFileCodec { List decode(byte[] encoded) { Map fields = fields(new String(encoded, StandardCharsets.UTF_8)); - if (!"1".equals(fields.remove("schema"))) { + if (!SCHEMA_VERSION.equals(fields.remove("schema"))) { throw new IllegalArgumentException("Unknown migration operation schema"); } int count = integer(fields.remove("count")); @@ -74,6 +76,8 @@ final class MigrationOperationFileCodec { field(output, prefix, "activation", Boolean.toString(value.activationAvailable())); field(output, prefix, "restart", Boolean.toString(value.restartRequired())); field(output, prefix, "external", Boolean.toString(value.externalApplyRequired())); + field(output, prefix, "targetIdentityHash", value.targetIdentityHash()); + field(output, prefix, "managedCandidateGeneration", optional(value.managedCandidateGeneration())); } private MigrationOperationSnapshot snapshot(Map fields, int index) { @@ -94,7 +98,9 @@ final class MigrationOperationFileCodec { Long.parseLong(take(fields, prefix, "pollMillis")), bool(take(fields, prefix, "activation")), bool(take(fields, prefix, "restart")), - bool(take(fields, prefix, "external"))); + bool(take(fields, prefix, "external")), + take(fields, prefix, "targetIdentityHash"), + nullable(take(fields, prefix, "managedCandidateGeneration"))); } private Map fields(String content) { @@ -135,6 +141,10 @@ final class MigrationOperationFileCodec { return ABSENT.equals(value) ? null : Instant.parse(value); } + private String nullable(String value) { + return ABSENT.equals(value) ? null : value; + } + private SetupErrorCode error(String value) { return ABSENT.equals(value) ? null : value(SetupErrorCode.class, value); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationSnapshot.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationSnapshot.java index 0b295a1673..900330f0d7 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationSnapshot.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationSnapshot.java @@ -7,8 +7,10 @@ package org.apache.hertzbeat.manager.setup.workflow; +import com.fasterxml.jackson.annotation.JsonIgnore; import java.time.Instant; import java.util.Objects; +import java.util.regex.Pattern; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; @@ -35,10 +37,16 @@ public record MigrationOperationSnapshot( long nextPollAfterMillis, boolean activationAvailable, boolean restartRequired, - boolean externalApplyRequired) { + boolean externalApplyRequired, + @JsonIgnore String targetIdentityHash, + @JsonIgnore String managedCandidateGeneration) { + + private static final Pattern TARGET_IDENTITY_HASH = Pattern.compile("[0-9a-f]{64}"); + private static final Pattern MANAGED_CANDIDATE_GENERATION = Pattern.compile("[A-Za-z0-9][A-Za-z0-9-]{0,63}"); public MigrationOperationSnapshot { Objects.requireNonNull(applyMode, "applyMode"); + validateManifestIdentity(applyMode, targetIdentityHash, managedCandidateGeneration); new MigrationView(operationId, state, MetadataDatabaseKind.H2, target, stage, progressPercent, createdAt, startedAt, completedAt, verificationState, errorCode, nextPollAfterMillis, activationAvailable, restartRequired, externalApplyRequired); @@ -51,6 +59,28 @@ public record MigrationOperationSnapshot( || state == MigrationOperationState.ROLLED_BACK; } + @Override + public String toString() { + return "MigrationOperationSnapshot[operationId=" + operationId + ", state=" + state + + ", target=" + target + ", applyMode=" + applyMode + ", stage=" + stage + + ", progressPercent=" + progressPercent + "]"; + } + + private static void validateManifestIdentity( + ApplyMode applyMode, String targetIdentityHash, String managedCandidateGeneration) { + if (targetIdentityHash == null || !TARGET_IDENTITY_HASH.matcher(targetIdentityHash).matches()) { + throw new IllegalArgumentException("Invalid migration target identity"); + } + if (applyMode == ApplyMode.MANAGED_WRITE) { + if (managedCandidateGeneration == null + || !MANAGED_CANDIDATE_GENERATION.matcher(managedCandidateGeneration).matches()) { + throw new IllegalArgumentException("Invalid managed migration candidate"); + } + } else if (managedCandidateGeneration != null) { + throw new IllegalArgumentException("External migration cannot reference a managed candidate"); + } + } + private static void validateRollback( MigrationOperationState state, MigrationStage stage, VerificationState verification, SetupErrorCode errorCode, MigrationRollbackOrigin origin) { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicy.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicy.java index d2892c6fa3..558a256648 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicy.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicy.java @@ -7,6 +7,7 @@ package org.apache.hertzbeat.manager.setup.workflow; +import java.util.Objects; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; @@ -20,6 +21,8 @@ final class MigrationOperationTransitionPolicy { if (!current.operationId().equals(next.operationId()) || current.target() != next.target() || current.applyMode() != next.applyMode() + || !current.targetIdentityHash().equals(next.targetIdentityHash()) + || !Objects.equals(current.managedCandidateGeneration(), next.managedCandidateGeneration()) || !current.createdAt().equals(next.createdAt()) || current.startedAt() != null && !current.startedAt().equals(next.startedAt()) || next.progressPercent() < current.progressPercent() diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStoreTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStoreTest.java index 3b399eee7d..8f65d25283 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStoreTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStoreTest.java @@ -10,6 +10,7 @@ package org.apache.hertzbeat.manager.setup.workflow; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.AtomicMoveNotSupportedException; @@ -36,6 +37,11 @@ import org.junit.jupiter.api.io.TempDir; class FileMigrationOperationStoreTest { + private static final String TARGET_IDENTITY_HASH = + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + private static final String MANAGED_CANDIDATE_GENERATION = "migration-generation-1"; + private final ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules(); + @TempDir private Path root; @@ -53,7 +59,49 @@ class FileMigrationOperationStoreTest { .isTrue(); String persisted = Files.readString(root.resolve(FileMigrationOperationStore.RELATIVE_PATH)); + assertThat(persisted).startsWith("schema=2\n") + .contains("targetIdentityHash=" + TARGET_IDENTITY_HASH) + .contains("managedCandidateGeneration=" + MANAGED_CANDIDATE_GENERATION); assertThat(persisted).doesNotContain("jdbc:", "username", "password", "SELECT", "secret-value"); + assertThat(running.toString()) + .doesNotContain(TARGET_IDENTITY_HASH, MANAGED_CANDIDATE_GENERATION, + "targetIdentityHash", "managedCandidateGeneration"); + assertThat(objectMapper.writeValueAsString(running)) + .doesNotContain(TARGET_IDENTITY_HASH, MANAGED_CANDIDATE_GENERATION, + "targetIdentityHash", "managedCandidateGeneration"); + } + + @Test + void roundTripsManagedAndExternalVersionTwoManifests() { + MigrationOperationSnapshot managed = succeeded( + pending("managed", Instant.parse("2026-08-09T01:00:00Z"))); + MigrationOperationSnapshot external = externalPending( + "external", Instant.parse("2026-08-09T02:00:00Z")); + + MigrationOperationFileCodec codec = new MigrationOperationFileCodec(); + byte[] encoded = codec.encode(List.of(managed, external)); + + assertThat(new String(encoded, StandardCharsets.UTF_8)).startsWith("schema=2\n"); + assertThat(codec.decode(encoded)).containsExactly(managed, external); + } + + @Test + void rejectsInvalidIdentityAndCandidateCouplingBeforePersistence() { + Instant createdAt = Instant.parse("2026-08-09T01:00:00Z"); + + assertThatThrownBy(() -> pending("invalid-hash", createdAt, + TARGET_IDENTITY_HASH.toUpperCase(), MANAGED_CANDIDATE_GENERATION)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> pending("missing-candidate", createdAt, TARGET_IDENTITY_HASH, null)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> pending("unsafe-candidate", createdAt, + TARGET_IDENTITY_HASH, "candidate/../secret")) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> pending("reserved-candidate", createdAt, TARGET_IDENTITY_HASH, "-")) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> externalPending( + "external-with-candidate", createdAt, MANAGED_CANDIDATE_GENERATION)) + .isInstanceOf(IllegalArgumentException.class); } @Test @@ -97,7 +145,11 @@ class FileMigrationOperationStoreTest { assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, () -> new FileMigrationOperationStore(root).history()); - Files.writeString(file, "schema=1\ncount=1\n", StandardCharsets.UTF_8); + MigrationOperationSnapshot pending = pending("old-schema", Instant.parse("2026-08-09T01:00:00Z")); + String oldSchema = new String( + new MigrationOperationFileCodec().encode(List.of(pending)), StandardCharsets.UTF_8) + .replaceFirst("schema=2", "schema=1"); + Files.writeString(file, oldSchema, StandardCharsets.UTF_8); ownerOnly(file); assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, () -> new FileMigrationOperationStore(root).history()); @@ -113,7 +165,7 @@ class FileMigrationOperationStoreTest { void rejectsNonOwnerOnlyFileAndSymlinkedConfigurationDirectory() throws Exception { Path file = root.resolve(FileMigrationOperationStore.RELATIVE_PATH); Files.createDirectories(file.getParent()); - Files.writeString(file, "schema=1\ncount=0\n", StandardCharsets.UTF_8); + Files.writeString(file, "schema=2\ncount=0\n", StandardCharsets.UTF_8); if (Files.getFileStore(file).supportsFileAttributeView("posix")) { Files.setPosixFilePermissions(file, PosixFilePermissions.fromString("rw-r--r--")); assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, @@ -191,6 +243,39 @@ class FileMigrationOperationStoreTest { () -> new FileMigrationOperationStore(root).history()); } + @Test + void missingOrTamperedVersionTwoIdentityFieldsRequireRecovery() throws Exception { + MigrationOperationSnapshot pending = pending("tampered", Instant.parse("2026-08-09T01:00:00Z")); + String encoded = new String( + new MigrationOperationFileCodec().encode(List.of(pending)), StandardCharsets.UTF_8); + Path file = root.resolve(FileMigrationOperationStore.RELATIVE_PATH); + + SecureSetupFile.create(root, file, encoded + .replace("0.targetIdentityHash=" + TARGET_IDENTITY_HASH + "\n", "") + .getBytes(StandardCharsets.UTF_8)); + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> new FileMigrationOperationStore(root).history()); + + Files.writeString(file, encoded.replace(TARGET_IDENTITY_HASH, TARGET_IDENTITY_HASH.toUpperCase()), + StandardCharsets.UTF_8); + ownerOnly(file); + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> new FileMigrationOperationStore(root).history()); + + Files.writeString(file, encoded.replace( + "0.managedCandidateGeneration=" + MANAGED_CANDIDATE_GENERATION, + "0.managedCandidateGeneration=-"), StandardCharsets.UTF_8); + ownerOnly(file); + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> new FileMigrationOperationStore(root).history()); + + Files.writeString(file, encoded.replace( + "0.applyMode=MANAGED_WRITE", "0.applyMode=EXTERNAL_APPLY"), StandardCharsets.UTF_8); + ownerOnly(file); + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> new FileMigrationOperationStore(root).history()); + } + @Test void failedAtomicPublicationPreservesPreviousState() throws Exception { FileMigrationOperationStore initial = new FileMigrationOperationStore(root); @@ -252,23 +337,42 @@ class FileMigrationOperationStoreTest { } private static MigrationOperationSnapshot pending(String id, Instant createdAt) { + return pending(id, createdAt, TARGET_IDENTITY_HASH, MANAGED_CANDIDATE_GENERATION); + } + + private static MigrationOperationSnapshot pending( + String id, Instant createdAt, String targetIdentityHash, String managedCandidateGeneration) { return new MigrationOperationSnapshot(id, MigrationOperationState.PENDING, MigrationTarget.MYSQL, ApplyMode.MANAGED_WRITE, MigrationStage.QUEUED, 0, createdAt, null, null, - VerificationState.PENDING, null, null, 1000, false, false, false); + VerificationState.PENDING, null, null, 1000, false, false, false, + targetIdentityHash, managedCandidateGeneration); + } + + private static MigrationOperationSnapshot externalPending(String id, Instant createdAt) { + return externalPending(id, createdAt, null); + } + + private static MigrationOperationSnapshot externalPending( + String id, Instant createdAt, String managedCandidateGeneration) { + return new MigrationOperationSnapshot(id, MigrationOperationState.PENDING, MigrationTarget.POSTGRESQL, + ApplyMode.EXTERNAL_APPLY, MigrationStage.QUEUED, 0, createdAt, null, null, + VerificationState.PENDING, null, null, 1000, false, false, false, + TARGET_IDENTITY_HASH, managedCandidateGeneration); } private static MigrationOperationSnapshot running(MigrationOperationSnapshot pending, int progress) { return new MigrationOperationSnapshot(pending.operationId(), MigrationOperationState.RUNNING, pending.target(), pending.applyMode(), MigrationStage.COPYING, progress, pending.createdAt(), pending.createdAt().plusSeconds(1), null, VerificationState.PENDING, null, null, 1000, - false, false, false); + false, false, false, pending.targetIdentityHash(), pending.managedCandidateGeneration()); } private static MigrationOperationSnapshot succeeded(MigrationOperationSnapshot pending) { Instant started = pending.createdAt().plusSeconds(1); return new MigrationOperationSnapshot(pending.operationId(), MigrationOperationState.SUCCEEDED, pending.target(), pending.applyMode(), MigrationStage.COMPLETED, 100, pending.createdAt(), started, - started.plusSeconds(1), VerificationState.SUCCEEDED, null, null, 0, false, false, false); + started.plusSeconds(1), VerificationState.SUCCEEDED, null, null, 0, false, false, false, + pending.targetIdentityHash(), pending.managedCandidateGeneration()); } private static MigrationOperationSnapshot rolledBack( @@ -277,7 +381,8 @@ class FileMigrationOperationStoreTest { return new MigrationOperationSnapshot(pending.operationId(), MigrationOperationState.ROLLED_BACK, pending.target(), pending.applyMode(), MigrationStage.ROLLED_BACK, 100, pending.createdAt(), started, started.plusSeconds(1), origin.verificationState(), - origin.errorCode(), origin, 0, false, false, false); + origin.errorCode(), origin, 0, false, false, false, + pending.targetIdentityHash(), pending.managedCandidateGeneration()); } private static void complete(FileMigrationOperationStore store, MigrationOperationSnapshot pending) { @@ -286,17 +391,20 @@ class FileMigrationOperationStoreTest { MigrationOperationSnapshot verifying = new MigrationOperationSnapshot( pending.operationId(), MigrationOperationState.RUNNING, pending.target(), pending.applyMode(), MigrationStage.VERIFYING, 100, pending.createdAt(), pending.createdAt().plusSeconds(1), null, - VerificationState.RUNNING, null, null, 1000, false, false, false); + VerificationState.RUNNING, null, null, 1000, false, false, false, + pending.targetIdentityHash(), pending.managedCandidateGeneration()); store.compareAndTransition(pending.operationId(), MigrationOperationState.RUNNING, verifying); MigrationOperationSnapshot ready = new MigrationOperationSnapshot( pending.operationId(), MigrationOperationState.READY_TO_ACTIVATE, pending.target(), pending.applyMode(), MigrationStage.READY_TO_ACTIVATE, 100, pending.createdAt(), pending.createdAt().plusSeconds(1), null, - VerificationState.SUCCEEDED, null, null, 0, true, false, false); + VerificationState.SUCCEEDED, null, null, 0, true, false, false, + pending.targetIdentityHash(), pending.managedCandidateGeneration()); store.compareAndTransition(pending.operationId(), MigrationOperationState.RUNNING, ready); MigrationOperationSnapshot activating = new MigrationOperationSnapshot( pending.operationId(), MigrationOperationState.RUNNING, pending.target(), pending.applyMode(), MigrationStage.ACTIVATING, 100, pending.createdAt(), pending.createdAt().plusSeconds(1), null, - VerificationState.SUCCEEDED, null, null, 1000, false, false, false); + VerificationState.SUCCEEDED, null, null, 1000, false, false, false, + pending.targetIdentityHash(), pending.managedCandidateGeneration()); store.compareAndTransition(pending.operationId(), MigrationOperationState.READY_TO_ACTIVATE, activating); store.compareAndTransition(pending.operationId(), MigrationOperationState.RUNNING, succeeded(pending)); } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicyTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicyTest.java index 07db9e74a3..37de72b5b7 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicyTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicyTest.java @@ -21,6 +21,9 @@ import org.junit.jupiter.api.Test; class MigrationOperationTransitionPolicyTest { + private static final String TARGET_IDENTITY_HASH = + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + private static final String MANAGED_CANDIDATE_GENERATION = "migration-generation-1"; private static final Instant CREATED = Instant.parse("2026-08-09T01:00:00Z"); private static final Instant STARTED = CREATED.plusSeconds(1); private static final Instant COMPLETED = STARTED.plusSeconds(1); @@ -172,13 +175,36 @@ class MigrationOperationTransitionPolicyTest { assertRejected(ready, verifying); } + @Test + void targetIdentityAndManagedCandidateStayImmutableAcrossEveryTransition() { + MigrationOperationSnapshot pending = snapshot(MigrationOperationState.PENDING, MigrationStage.QUEUED, + 0, null, null, VerificationState.PENDING, null, 1000, false, false, false); + MigrationOperationSnapshot running = snapshot(MigrationOperationState.RUNNING, MigrationStage.COPYING, + 25, STARTED, null, VerificationState.PENDING, null, 1000, false, false, false); + + assertRejected(pending, new MigrationOperationSnapshot( + running.operationId(), running.state(), running.target(), running.applyMode(), running.stage(), + running.progressPercent(), running.createdAt(), running.startedAt(), running.completedAt(), + running.verificationState(), running.errorCode(), running.rollbackOrigin(), + running.nextPollAfterMillis(), running.activationAvailable(), running.restartRequired(), + running.externalApplyRequired(), "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + running.managedCandidateGeneration())); + assertRejected(pending, new MigrationOperationSnapshot( + running.operationId(), running.state(), running.target(), running.applyMode(), running.stage(), + running.progressPercent(), running.createdAt(), running.startedAt(), running.completedAt(), + running.verificationState(), running.errorCode(), running.rollbackOrigin(), + running.nextPollAfterMillis(), running.activationAvailable(), running.restartRequired(), + running.externalApplyRequired(), running.targetIdentityHash(), "migration-generation-2")); + } + private MigrationOperationSnapshot external( MigrationOperationState state, MigrationStage stage, VerificationState verification, SetupErrorCode error, boolean externalRequired, Instant completedAt) { return new MigrationOperationSnapshot("migration-1", state, MigrationTarget.POSTGRESQL, ApplyMode.EXTERNAL_APPLY, stage, 100, CREATED, STARTED, completedAt, verification, error, null, state == MigrationOperationState.RUNNING ? 1000 : 0, - state == MigrationOperationState.READY_TO_ACTIVATE, false, externalRequired); + state == MigrationOperationState.READY_TO_ACTIVATE, false, externalRequired, + TARGET_IDENTITY_HASH, null); } private MigrationOperationSnapshot snapshot( @@ -187,7 +213,8 @@ class MigrationOperationTransitionPolicyTest { boolean activation, boolean restart, boolean external) { return new MigrationOperationSnapshot("migration-1", state, MigrationTarget.MYSQL, ApplyMode.MANAGED_WRITE, stage, progress, CREATED, startedAt, completedAt, - verification, error, null, pollMillis, activation, restart, external); + verification, error, null, pollMillis, activation, restart, external, + TARGET_IDENTITY_HASH, MANAGED_CANDIDATE_GENERATION); } private MigrationOperationSnapshot failed(SetupErrorCode errorCode) { @@ -205,7 +232,7 @@ class MigrationOperationTransitionPolicyTest { return new MigrationOperationSnapshot("migration-1", MigrationOperationState.RUNNING, MigrationTarget.MYSQL, ApplyMode.MANAGED_WRITE, MigrationStage.ROLLING_BACK, 100, CREATED, STARTED, null, origin.verificationState(), null, origin, - 1000, false, false, false); + 1000, false, false, false, TARGET_IDENTITY_HASH, MANAGED_CANDIDATE_GENERATION); } private MigrationOperationSnapshot rolledBack( @@ -213,7 +240,7 @@ class MigrationOperationTransitionPolicyTest { return new MigrationOperationSnapshot("migration-1", MigrationOperationState.ROLLED_BACK, MigrationTarget.MYSQL, ApplyMode.MANAGED_WRITE, MigrationStage.ROLLED_BACK, 100, CREATED, STARTED, COMPLETED, origin.verificationState(), errorCode, origin, - 0, false, false, false); + 0, false, false, false, TARGET_IDENTITY_HASH, MANAGED_CANDIDATE_GENERATION); } private void assertAllowed(MigrationOperationSnapshot current, MigrationOperationSnapshot next) { From c4190b4c0ab515c0e43ba436bf92e198b7952fbf Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 06:47:40 +0800 Subject: [PATCH 42/71] Persist metadata migration candidates --- .../ManagedConfigDeploymentDetector.java | 9 +- .../config/ManagedConfigurationLock.java | 30 ++ .../ManagedConfigurationTransaction.java | 38 +- ...agedMigrationConfigurationTransaction.java | 130 +++++ .../config/MigrationCandidateManifest.java | 18 + .../MigrationCandidateManifestCodec.java | 73 +++ .../config/MigrationCandidateMaterial.java | 45 ++ .../setup/config/MigrationCandidateStore.java | 266 ++++++++++ .../setup/security/SecureSetupFileLock.java | 27 +- .../ManagedConfigurationTransactionTest.java | 4 +- .../ManagedDeploymentCapabilityTest.java | 61 ++- ...MigrationConfigurationTransactionTest.java | 493 ++++++++++++++++++ 12 files changed, 1154 insertions(+), 40 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationLock.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransaction.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateManifest.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateManifestCodec.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateMaterial.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateStore.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransactionTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigDeploymentDetector.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigDeploymentDetector.java index b43d009e62..337e9ab1a1 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigDeploymentDetector.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigDeploymentDetector.java @@ -21,6 +21,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.function.Predicate; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFileLock; /** Detects whether setup may safely use managed files or must export them for an operator. */ public final class ManagedConfigDeploymentDetector { @@ -32,7 +33,7 @@ public final class ManagedConfigDeploymentDetector { "managed-secrets.properties", "managed-secrets.properties.candidate", "managed-secrets.properties.last-known-good", - ".managed-config.lock"); + ManagedConfigurationLock.LOCK_FILE_NAME); private final Path installationRoot; private final Predicate writable; @@ -76,6 +77,12 @@ public final class ManagedConfigDeploymentDetector { if (Files.exists(managedFile) && !Files.isRegularFile(managedFile)) { return ManagedConfigCapability.constrained(DeploymentConstraint.UNSAFE_PATH); } + if (fileName.equals(ManagedConfigurationLock.LOCK_FILE_NAME) + && Files.exists(managedFile) + && !SecureSetupFileLock.isValidExistingLock( + installationRoot, "data/config/" + ManagedConfigurationLock.LOCK_FILE_NAME)) { + return ManagedConfigCapability.constrained(DeploymentConstraint.UNSAFE_PATH); + } if (Files.exists(managedFile) && (!readable.test(managedFile) || !writable.test(managedFile))) { return ManagedConfigCapability.constrained(DeploymentConstraint.READ_ONLY); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationLock.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationLock.java new file mode 100644 index 0000000000..db6361279b --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationLock.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.io.IOException; +import java.nio.file.Path; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFileLock; + +/** Shared lock boundary for every writer of the managed configuration pair. */ +final class ManagedConfigurationLock { + + static final String LOCK_FILE_NAME = ".managed-config-v2.lock"; + + private static final String LOCK_FILE = "data/config/" + LOCK_FILE_NAME; + + private final SecureSetupFileLock delegate; + + ManagedConfigurationLock(Path installationRoot) { + delegate = new SecureSetupFileLock(installationRoot, LOCK_FILE); + } + + T execute(SecureSetupFileLock.IoOperation operation) throws IOException { + return delegate.execute(operation); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransaction.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransaction.java index 84dce63e6c..f9050932ff 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransaction.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransaction.java @@ -18,12 +18,7 @@ package org.apache.hertzbeat.manager.setup.config; import java.io.IOException; -import java.nio.channels.FileChannel; -import java.nio.channels.FileLock; -import java.nio.channels.OverlappingFileLockException; -import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; import java.util.Objects; import java.util.Optional; import java.util.UUID; @@ -31,11 +26,9 @@ import java.util.UUID; /** Coordinates the application and secret snapshots as one locked, recoverable operation. */ public final class ManagedConfigurationTransaction { - private static final String LOCK_FILE = ".managed-config.lock"; - private final ManagedApplicationConfigStore applicationStore; private final ManagedSecretStore secretStore; - private final Path lockFile; + private final ManagedConfigurationLock lock; private final ManagedConfigurationRecovery recovery; private final RecoveryFailureReporter reporter; @@ -62,9 +55,8 @@ public final class ManagedConfigurationTransaction { this.secretStore = Objects.requireNonNull(secretStore, "secretStore"); this.reporter = Objects.requireNonNull(reporter, "reporter"); this.recovery = new ManagedConfigurationRecovery(applicationStore, secretStore, reporter); - Path root = Objects.requireNonNull(installationRoot, "installationRoot") - .toAbsolutePath().normalize(); - this.lockFile = root.resolve("data/config").resolve(LOCK_FILE); + this.lock = new ManagedConfigurationLock( + Objects.requireNonNull(installationRoot, "installationRoot")); } /** Stages and publishes one validated configuration generation under the process lock. */ @@ -136,29 +128,7 @@ public final class ManagedConfigurationTransaction { } private Outcome withLock(LockedOperation operation) throws IOException { - Path directory = lockFile.getParent(); - if (Files.isSymbolicLink(lockFile) || Files.isSymbolicLink(directory) - || Files.isSymbolicLink(directory.getParent()) - || Files.isSymbolicLink(directory.getParent().getParent())) { - throw new IOException("Managed configuration lock is unavailable"); - } - Files.createDirectories(directory); - try (FileChannel channel = FileChannel.open(lockFile, - StandardOpenOption.CREATE, StandardOpenOption.WRITE); - FileLock lock = tryLock(channel)) { - if (lock == null) { - throw new IOException("Managed configuration operation is already in progress"); - } - return operation.run(); - } - } - - private static FileLock tryLock(FileChannel channel) throws IOException { - try { - return channel.tryLock(); - } catch (OverlappingFileLockException failure) { - return null; - } + return lock.execute(operation::run); } private static void discardCandidate(ManagedApplicationConfigStore store, IOException failure) { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransaction.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransaction.java new file mode 100644 index 0000000000..c2323a84d5 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransaction.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Objects; +import java.util.Optional; +import java.util.regex.Pattern; +import org.apache.hertzbeat.manager.setup.api.OperationIdValidator; + +/** Locked public boundary for migration-owned managed-configuration candidates. */ +public final class ManagedMigrationConfigurationTransaction { + + private static final Pattern GENERATION = Pattern.compile("[A-Za-z0-9][A-Za-z0-9-]{0,63}"); + private static final Pattern IDENTITY_HASH = Pattern.compile("[0-9a-f]{64}"); + + private final ManagedConfigurationLock lock; + private final MigrationCandidateStore store; + + /** Creates the production migration candidate transaction. */ + public ManagedMigrationConfigurationTransaction(Path installationRoot) { + lock = new ManagedConfigurationLock(installationRoot); + store = new MigrationCandidateStore(installationRoot); + } + + /** Stages the exact candidate or fails with a stable, secret-free error. */ + public CandidateRef stage(String operationId, String candidateGeneration, String baseGeneration, + String targetIdentityHash, ManagedConfigurationBundle bundle) throws IOException { + StageOutcome outcome = stageOutcome( + operationId, candidateGeneration, baseGeneration, targetIdentityHash, bundle); + if (outcome != StageOutcome.STAGED && outcome != StageOutcome.ALREADY_STAGED) { + throw new IOException("Managed migration candidate was not staged: " + outcome); + } + return new CandidateRef(operationId, candidateGeneration); + } + + /** Stages without changing active, last-known-good, or setup-owned candidates. */ + public StageOutcome stageOutcome(String operationId, String candidateGeneration, String baseGeneration, + String targetIdentityHash, ManagedConfigurationBundle bundle) + throws IOException { + CandidateRef reference = new CandidateRef(operationId, candidateGeneration); + requireGeneration(baseGeneration, "base generation"); + requireIdentityHash(targetIdentityHash); + Objects.requireNonNull(bundle, "bundle"); + return lock.execute(() -> store.stage(reference, baseGeneration, targetIdentityHash, bundle)); + } + + /** Returns secret-free exact metadata; corrupt or partial candidates fail closed. */ + public Inspection inspect(CandidateRef reference) throws IOException { + Objects.requireNonNull(reference, "reference"); + return lock.execute(() -> store.inspect(reference)); + } + + /** Provides synchronous, non-retaining access and clears decoded secrets on every exit. */ + public T readExact(CandidateRef reference, CandidateReader reader) throws IOException { + Objects.requireNonNull(reference, "reference"); + Objects.requireNonNull(reader, "reader"); + return lock.execute(() -> store.readExact(reference, reader)); + } + + /** Removes only the operation-and-generation scoped candidate named by the reference. */ + public DiscardOutcome discardExact(CandidateRef reference) throws IOException { + Objects.requireNonNull(reference, "reference"); + return lock.execute(() -> store.discardExact(reference)); + } + + static void requireGeneration(String value, String label) { + Objects.requireNonNull(value, label); + if (!GENERATION.matcher(value).matches()) { + throw new IllegalArgumentException("Invalid managed migration " + label); + } + } + + static void requireIdentityHash(String value) { + Objects.requireNonNull(value, "targetIdentityHash"); + if (!IDENTITY_HASH.matcher(value).matches()) { + throw new IllegalArgumentException("Invalid managed migration target identity"); + } + } + + /** Exact public handle; contains neither configuration paths nor target credentials. */ + public record CandidateRef(String operationId, String candidateGeneration) { + public CandidateRef { + if (!OperationIdValidator.isSafe(operationId)) { + throw new IllegalArgumentException("Invalid managed migration operation id"); + } + requireGeneration(candidateGeneration, "candidate generation"); + } + } + + /** Secret-free persisted candidate state and exact base/target identity metadata. */ + public record Inspection(CandidateState state, Optional baseGeneration, + Optional targetIdentityHash) { + public Inspection { + Objects.requireNonNull(state, "state"); + Objects.requireNonNull(baseGeneration, "baseGeneration"); + Objects.requireNonNull(targetIdentityHash, "targetIdentityHash"); + if ((state == CandidateState.READY) != baseGeneration.isPresent() + || baseGeneration.isPresent() != targetIdentityHash.isPresent()) { + throw new IllegalArgumentException("Only a ready candidate exposes exact metadata"); + } + } + + @Override + public String toString() { + return "Inspection[state=" + state + ", exactMetadata=" + baseGeneration.isPresent() + "]"; + } + } + + /** Persisted completeness state. */ + public enum CandidateState { MISSING, READY, RECOVERY_REQUIRED } + + /** Stable staging result without filesystem or configuration details. */ + public enum StageOutcome { STAGED, ALREADY_STAGED, STALE, RECOVERY_REQUIRED } + + /** Stable exact-discard result. */ + public enum DiscardOutcome { DISCARDED, NOT_FOUND } + + /** Synchronous, non-retaining access to a decoded candidate bundle. */ + @FunctionalInterface + public interface CandidateReader { + T read(ManagedConfigurationBundle bundle); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateManifest.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateManifest.java new file mode 100644 index 0000000000..e9e0c1c753 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateManifest.java @@ -0,0 +1,18 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +/** Immutable, secret-free identity recorded beside a migration candidate bundle. */ +record MigrationCandidateManifest(String operationId, String candidateGeneration, + String baseGeneration, String targetIdentityHash) { + void validate() { + new ManagedMigrationConfigurationTransaction.CandidateRef(operationId, candidateGeneration); + ManagedMigrationConfigurationTransaction.requireGeneration(baseGeneration, "base generation"); + ManagedMigrationConfigurationTransaction.requireIdentityHash(targetIdentityHash); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateManifestCodec.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateManifestCodec.java new file mode 100644 index 0000000000..17472460c6 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateManifestCodec.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; + +/** Checksum-protected, secret-free codec for one exact migration candidate identity. */ +final class MigrationCandidateManifestCodec { + + private static final String HEADER = "hertzbeat-managed-migration-candidate=1\n"; + private static final String CHECKSUM_PREFIX = "sha256="; + + byte[] encode(MigrationCandidateManifest manifest) { + String body = HEADER + + "operationId=" + manifest.operationId() + "\n" + + "candidateGeneration=" + manifest.candidateGeneration() + "\n" + + "baseGeneration=" + manifest.baseGeneration() + "\n" + + "targetIdentityHash=" + manifest.targetIdentityHash() + "\n"; + return (body + CHECKSUM_PREFIX + checksum(body) + "\n").getBytes(StandardCharsets.US_ASCII); + } + + MigrationCandidateManifest decode(byte[] encoded) throws IOException { + String document = new String(encoded, StandardCharsets.US_ASCII); + int checksumStart = document.lastIndexOf(CHECKSUM_PREFIX); + if (checksumStart < 0 || !document.endsWith("\n")) { + throw invalid(); + } + String body = document.substring(0, checksumStart); + String expected = document.substring(checksumStart + CHECKSUM_PREFIX.length(), document.length() - 1); + if (!MessageDigest.isEqual(expected.getBytes(StandardCharsets.US_ASCII), + checksum(body).getBytes(StandardCharsets.US_ASCII))) { + throw invalid(); + } + String[] lines = body.split("\n", -1); + if (lines.length != 6 || !(lines[0] + "\n").equals(HEADER)) { + throw invalid(); + } + MigrationCandidateManifest manifest = new MigrationCandidateManifest( + value(lines[1], "operationId="), value(lines[2], "candidateGeneration="), + value(lines[3], "baseGeneration="), value(lines[4], "targetIdentityHash=")); + manifest.validate(); + return manifest; + } + + private static String value(String line, String prefix) throws IOException { + if (!line.startsWith(prefix)) { + throw invalid(); + } + return line.substring(prefix.length()); + } + + private static String checksum(String content) { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(content.getBytes(StandardCharsets.US_ASCII))); + } catch (NoSuchAlgorithmException impossible) { + throw new IllegalStateException("SHA-256 is unavailable", impossible); + } + } + + private static IOException invalid() { + return new IOException("Managed migration manifest is invalid"); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateMaterial.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateMaterial.java new file mode 100644 index 0000000000..c21708675c --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateMaterial.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.util.Optional; + +/** Closeable decoded material kept internal to one synchronous candidate operation. */ +record MigrationCandidateMaterial( + ManagedMigrationConfigurationTransaction.Inspection inspection, + Optional manifest, + Optional application, + Optional secrets) implements AutoCloseable { + + static MigrationCandidateMaterial missing() { + return empty(ManagedMigrationConfigurationTransaction.CandidateState.MISSING); + } + + static MigrationCandidateMaterial recoveryRequired() { + return empty(ManagedMigrationConfigurationTransaction.CandidateState.RECOVERY_REQUIRED); + } + + static MigrationCandidateMaterial ready(MigrationCandidateManifest manifest, + ManagedApplicationConfig application, ManagedSecrets secrets) { + return new MigrationCandidateMaterial(new ManagedMigrationConfigurationTransaction.Inspection( + ManagedMigrationConfigurationTransaction.CandidateState.READY, + Optional.of(manifest.baseGeneration()), Optional.of(manifest.targetIdentityHash())), + Optional.of(manifest), Optional.of(application), Optional.of(secrets)); + } + + private static MigrationCandidateMaterial empty( + ManagedMigrationConfigurationTransaction.CandidateState state) { + return new MigrationCandidateMaterial(new ManagedMigrationConfigurationTransaction.Inspection( + state, Optional.empty(), Optional.empty()), Optional.empty(), Optional.empty(), Optional.empty()); + } + + @Override + public void close() { + secrets.ifPresent(ManagedSecrets::close); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateStore.java new file mode 100644 index 0000000000..5cf2b08e99 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateStore.java @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Objects; +import java.util.UUID; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; + +/** Root-bound file store for generation-scoped, owner-only migration candidates. */ +final class MigrationCandidateStore { + + private static final int MAXIMUM_APPLICATION_BYTES = 1024 * 1024; + private static final int MAXIMUM_SECRET_BYTES = 256 * 1024; + private static final int MAXIMUM_MANIFEST_BYTES = 1024; + + private final Path root; + private final Path candidateRoot; + private final ManagedApplicationConfigStore applicationStore; + private final ManagedSecretStore secretStore; + private final ApplicationConfigDocumentCodec applicationCodec = new ApplicationConfigDocumentCodec(); + private final SecretConfigDocumentCodec secretCodec = new SecretConfigDocumentCodec(); + private final MigrationCandidateManifestCodec manifestCodec = new MigrationCandidateManifestCodec(); + + MigrationCandidateStore(Path installationRoot) { + root = prepareRoot(installationRoot); + candidateRoot = root.resolve("data/config/migration-candidates"); + applicationStore = new FileManagedApplicationConfigStore(root); + secretStore = new FileManagedSecretStore(root); + } + + ManagedMigrationConfigurationTransaction.StageOutcome stage( + ManagedMigrationConfigurationTransaction.CandidateRef reference, String baseGeneration, + String targetIdentityHash, ManagedConfigurationBundle bundle) throws IOException { + try (MigrationCandidateMaterial existing = read(reference)) { + if (existing.inspection().state() == ManagedMigrationConfigurationTransaction.CandidateState.READY) { + boolean same = existing.manifest().filter(manifest -> manifest.baseGeneration().equals(baseGeneration) + && manifest.targetIdentityHash().equals(targetIdentityHash)).isPresent() + && existing.application().filter(bundle.application()::equals).isPresent() + && existing.secrets().filter(bundle.secrets()::equals).isPresent(); + return same ? ManagedMigrationConfigurationTransaction.StageOutcome.ALREADY_STAGED + : ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED; + } + if (existing.inspection().state() + != ManagedMigrationConfigurationTransaction.CandidateState.MISSING) { + return ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED; + } + } + ActivePairState activePair = activePairState(baseGeneration); + if (activePair != ActivePairState.MATCH) { + return activePair == ActivePairState.STALE + ? ManagedMigrationConfigurationTransaction.StageOutcome.STALE + : ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED; + } + CandidatePaths paths = paths(reference); + byte[] application = applicationCodec.encode(bundle.application(), reference.candidateGeneration()); + byte[] secrets = secretCodec.encode(bundle.secrets(), reference.candidateGeneration()); + byte[] manifest = manifestCodec.encode(new MigrationCandidateManifest( + reference.operationId(), reference.candidateGeneration(), baseGeneration, targetIdentityHash)); + try { + publish(paths.application(), application); + publish(paths.secrets(), secrets); + publish(paths.manifest(), manifest); + try (MigrationCandidateMaterial staged = read(reference)) { + return staged.inspection().state() == ManagedMigrationConfigurationTransaction.CandidateState.READY + ? ManagedMigrationConfigurationTransaction.StageOutcome.STAGED + : ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED; + } + } finally { + clear(application); + clear(secrets); + clear(manifest); + } + } + + ManagedMigrationConfigurationTransaction.Inspection inspect( + ManagedMigrationConfigurationTransaction.CandidateRef reference) { + try (MigrationCandidateMaterial material = read(reference)) { + return material.inspection(); + } + } + + T readExact(ManagedMigrationConfigurationTransaction.CandidateRef reference, + ManagedMigrationConfigurationTransaction.CandidateReader reader) throws IOException { + try (MigrationCandidateMaterial material = read(reference)) { + if (material.inspection().state() != ManagedMigrationConfigurationTransaction.CandidateState.READY) { + throw new IOException("Managed migration candidate is not ready"); + } + ManagedConfigurationBundle bundle = new ManagedConfigurationBundle( + material.application().orElseThrow(), material.secrets().orElseThrow()); + return reader.read(bundle); + } + } + + ManagedMigrationConfigurationTransaction.DiscardOutcome discardExact( + ManagedMigrationConfigurationTransaction.CandidateRef reference) throws IOException { + CandidatePaths paths = paths(reference); + EntryState application = entryState(paths.application()); + EntryState secrets = entryState(paths.secrets()); + EntryState manifest = entryState(paths.manifest()); + if (application == EntryState.UNSAFE || secrets == EntryState.UNSAFE || manifest == EntryState.UNSAFE) { + throw new IOException("Managed migration candidate is unsafe"); + } + if (application == EntryState.MISSING && secrets == EntryState.MISSING && manifest == EntryState.MISSING) { + return ManagedMigrationConfigurationTransaction.DiscardOutcome.NOT_FOUND; + } + delete(paths.manifest()); + delete(paths.secrets()); + delete(paths.application()); + return ManagedMigrationConfigurationTransaction.DiscardOutcome.DISCARDED; + } + + private ActivePairState activePairState(String baseGeneration) { + CandidateRead application = applicationStore.readActive(); + CandidateRead secrets = secretStore.readActive(); + try { + if (!ManagedConfigurationTransaction.validPair(application, secrets)) { + return ActivePairState.RECOVERY_REQUIRED; + } + try { + new ManagedConfigurationBundle( + application.value().orElseThrow(), secrets.value().orElseThrow()); + } catch (IllegalArgumentException failure) { + return ActivePairState.RECOVERY_REQUIRED; + } + return application.generation().filter(baseGeneration::equals).isPresent() + ? ActivePairState.MATCH : ActivePairState.STALE; + } finally { + ManagedConfigurationTransaction.close(secrets); + } + } + + private MigrationCandidateMaterial read(ManagedMigrationConfigurationTransaction.CandidateRef reference) { + CandidatePaths paths = paths(reference); + EntryState application = entryState(paths.application()); + EntryState secrets = entryState(paths.secrets()); + EntryState manifest = entryState(paths.manifest()); + if (application == EntryState.UNSAFE || secrets == EntryState.UNSAFE || manifest == EntryState.UNSAFE) { + return MigrationCandidateMaterial.recoveryRequired(); + } + if (application == EntryState.MISSING && secrets == EntryState.MISSING && manifest == EntryState.MISSING) { + return MigrationCandidateMaterial.missing(); + } + if (application != EntryState.PRESENT || secrets != EntryState.PRESENT || manifest != EntryState.PRESENT) { + return MigrationCandidateMaterial.recoveryRequired(); + } + return decode(reference, paths); + } + + private MigrationCandidateMaterial decode(ManagedMigrationConfigurationTransaction.CandidateRef reference, + CandidatePaths paths) { + byte[] applicationBytes = null; + byte[] secretBytes = null; + byte[] manifestBytes = null; + ManagedDocumentCodec.Decoded decodedSecrets = null; + try { + applicationBytes = SecureSetupFile.readOwnerOnlyWithoutLinks( + root, paths.application(), MAXIMUM_APPLICATION_BYTES); + secretBytes = SecureSetupFile.readOwnerOnlyWithoutLinks(root, paths.secrets(), MAXIMUM_SECRET_BYTES); + manifestBytes = SecureSetupFile.readOwnerOnlyWithoutLinks(root, paths.manifest(), MAXIMUM_MANIFEST_BYTES); + ManagedDocumentCodec.Decoded decodedApplication = + applicationCodec.decode(applicationBytes); + decodedSecrets = secretCodec.decode(secretBytes); + MigrationCandidateManifest manifest = manifestCodec.decode(manifestBytes); + if (!manifest.operationId().equals(reference.operationId()) + || !manifest.candidateGeneration().equals(reference.candidateGeneration()) + || !decodedApplication.generation().equals(reference.candidateGeneration()) + || !decodedSecrets.generation().equals(reference.candidateGeneration())) { + return MigrationCandidateMaterial.recoveryRequired(); + } + ManagedSecrets transferred = decodedSecrets.value(); + new ManagedConfigurationBundle(decodedApplication.value(), transferred); + decodedSecrets = null; + return MigrationCandidateMaterial.ready(manifest, decodedApplication.value(), transferred); + } catch (IOException | ManagedDocumentCodec.DocumentException | IllegalArgumentException failure) { + return MigrationCandidateMaterial.recoveryRequired(); + } finally { + clear(applicationBytes); + clear(secretBytes); + clear(manifestBytes); + close(decodedSecrets == null ? null : decodedSecrets.value()); + } + } + + private CandidatePaths paths(ManagedMigrationConfigurationTransaction.CandidateRef reference) { + Path directory = candidateRoot.resolve(reference.operationId()) + .resolve(reference.candidateGeneration()).normalize(); + if (!directory.startsWith(candidateRoot)) { + throw new IllegalArgumentException("Managed migration candidate is invalid"); + } + return new CandidatePaths(directory.resolve("application"), directory.resolve("secrets"), + directory.resolve("manifest")); + } + + private void publish(Path target, byte[] content) throws IOException { + Path temporary = target.resolveSibling("." + target.getFileName() + "-" + UUID.randomUUID() + ".tmp"); + try { + SecureSetupFile.create(root, temporary, content); + SecureSetupFile.atomicReplace(root, temporary, target); + } finally { + if (Files.exists(temporary, LinkOption.NOFOLLOW_LINKS)) { + SecureSetupFile.deleteOwnerOnlyInsideRoot(root, temporary); + } + } + } + + private void delete(Path target) throws IOException { + if (Files.exists(target, LinkOption.NOFOLLOW_LINKS)) { + SecureSetupFile.deleteOwnerOnlyInsideRoot(root, target); + SecureSetupFile.forceParentDirectoryIfSupported(root, target); + } + } + + private EntryState entryState(Path path) { + boolean entryExists = Files.exists(path, LinkOption.NOFOLLOW_LINKS); + try { + boolean secureEntry = SecureSetupFile.existsInsideRootWithoutLinks(root, path); + if (entryExists && !secureEntry) { + return EntryState.UNSAFE; + } + return secureEntry ? EntryState.PRESENT : EntryState.MISSING; + } catch (IOException failure) { + return EntryState.UNSAFE; + } + } + + private static Path prepareRoot(Path installationRoot) { + try { + return SecureSetupFile.prepareTrustedRoot(Objects.requireNonNull(installationRoot, "installationRoot")); + } catch (IOException failure) { + throw new IllegalArgumentException("Managed migration root is unsafe"); + } + } + + private static void close(Object value) { + if (value instanceof AutoCloseable closeable) { + try { + closeable.close(); + } catch (Exception ignored) { + // Secret cleanup must not replace the persistence result. + } + } + } + + private static void clear(byte[] content) { + if (content != null) { + Arrays.fill(content, (byte) 0); + } + } + + private record CandidatePaths(Path application, Path secrets, Path manifest) { } + + private enum ActivePairState { MATCH, STALE, RECOVERY_REQUIRED } + + private enum EntryState { MISSING, PRESENT, UNSAFE } + +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileLock.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileLock.java index 831af95c2c..6aaa58e26a 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileLock.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileLock.java @@ -18,6 +18,7 @@ import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.BasicFileAttributes; +import java.util.Arrays; import java.util.Objects; import java.util.Set; import java.util.UUID; @@ -75,6 +76,30 @@ public final class SecureSetupFileLock { }); } + /** Validates an existing lock without creating, replacing, or otherwise mutating it. */ + public static boolean isValidExistingLock(Path installationRoot, String relativePath) { + Objects.requireNonNull(installationRoot, "installationRoot"); + Objects.requireNonNull(relativePath, "relativePath"); + byte[] encoded = null; + try { + Path root = installationRoot.toAbsolutePath().normalize().toRealPath(); + Path lock = root.resolve(relativePath).normalize(); + if (!lock.startsWith(root) || lock.equals(root) + || !SecureSetupFile.isOwnerOnlyRegularFile(lock)) { + return false; + } + encoded = SecureSetupFile.readOwnerOnlyWithoutLinks(root, lock, MAXIMUM_IDENTITY_BYTES); + validateIdentity(new String(encoded, StandardCharsets.UTF_8)); + return true; + } catch (IOException | RuntimeException failure) { + return false; + } finally { + if (encoded != null) { + Arrays.fill(encoded, (byte) 0); + } + } + } + private LockIdentity initializeAndValidate() throws IOException { try { String created = IDENTITY_PREFIX + UUID.randomUUID() + '\n'; @@ -135,7 +160,7 @@ public final class SecureSetupFileLock { return validateIdentity(new String(encoded.array(), StandardCharsets.UTF_8)); } - private String validateIdentity(String encoded) throws IOException { + private static String validateIdentity(String encoded) throws IOException { String identity = encoded.strip(); if (!identity.startsWith(IDENTITY_PREFIX)) { throw new IOException("Secure setup-file lock identity is invalid"); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java index 6fda1c2ea6..61b19de622 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java @@ -159,7 +159,7 @@ class ManagedConfigurationTransactionTest { @Test void secondTransactionCannotEnterWhileSameProcessHoldsTheOsLock() throws Exception { Path config = Files.createDirectories(installationRoot.resolve("data/config")); - Path lockPath = config.resolve(".managed-config.lock"); + Path lockPath = config.resolve(".managed-config-v2.lock"); try (FileChannel channel = FileChannel.open(lockPath, StandardOpenOption.CREATE, StandardOpenOption.WRITE); FileLock ignored = channel.lock()) { @@ -171,7 +171,7 @@ class ManagedConfigurationTransactionTest { @Test void transactionCannotEnterWhileAnotherProcessHoldsTheOsLock() throws Exception { Path config = Files.createDirectories(installationRoot.resolve("data/config")); - Path lockPath = config.resolve(".managed-config.lock"); + Path lockPath = config.resolve(".managed-config-v2.lock"); Path ready = installationRoot.resolve("lock-ready"); Path release = installationRoot.resolve("lock-release"); Process holder = new ProcessBuilder( diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedDeploymentCapabilityTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedDeploymentCapabilityTest.java index aa61d42105..c6a5790d86 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedDeploymentCapabilityTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedDeploymentCapabilityTest.java @@ -22,15 +22,22 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermission; +import java.util.Set; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; class ManagedDeploymentCapabilityTest { + private static final byte[] VALID_LOCK_IDENTITY = + "secure-setup-lock-v1:01234567-89ab-cdef-0123-456789abcdef\n".getBytes(StandardCharsets.UTF_8); + @TempDir private Path installationRoot; @@ -127,7 +134,7 @@ class ManagedDeploymentCapabilityTest { void rejectsSymlinkedLockAndNonRegularSnapshotArtifacts() throws Exception { Path config = Files.createDirectories(installationRoot.resolve("data/config")); Path outside = Files.writeString(installationRoot.resolve("outside-lock"), "lock"); - Path lock = config.resolve(".managed-config.lock"); + Path lock = config.resolve(".managed-config-v2.lock"); try { Files.createSymbolicLink(lock, outside); } catch (UnsupportedOperationException | IOException exception) { @@ -145,7 +152,8 @@ class ManagedDeploymentCapabilityTest { @Test void checksLockCandidateAndLastKnownGoodReadWriteAccess() throws Exception { Path config = Files.createDirectories(installationRoot.resolve("data/config")); - Path lock = Files.writeString(config.resolve(".managed-config.lock"), "lock"); + Path lock = config.resolve(".managed-config-v2.lock"); + SecureSetupFile.create(installationRoot, lock, VALID_LOCK_IDENTITY); Path candidate = Files.writeString( config.resolve("managed-application.yml.candidate"), "candidate"); Path lastKnownGood = Files.writeString( @@ -161,4 +169,53 @@ class ManagedDeploymentCapabilityTest { new ManagedConfigDeploymentDetector( installationRoot, path -> true, path -> !path.equals(lastKnownGood)).detect().constraint()); } + + @Test + void ignoresTheLegacyLockButChecksTheVersionedLockShapeAndAccess() throws Exception { + Path config = Files.createDirectories(installationRoot.resolve("data/config")); + Path legacyLock = Files.write(config.resolve(".managed-config.lock"), new byte[0]); + Path versionedLock = config.resolve(".managed-config-v2.lock"); + + assertEquals(ApplyMode.MANAGED_WRITE, + new ManagedConfigDeploymentDetector( + installationRoot, path -> !path.equals(legacyLock), path -> !path.equals(legacyLock)) + .detect().applyMode()); + + Files.createDirectory(versionedLock); + assertEquals(DeploymentConstraint.UNSAFE_PATH, + new ManagedConfigDeploymentDetector(installationRoot).detect().constraint()); + Files.delete(versionedLock); + SecureSetupFile.create(installationRoot, versionedLock, VALID_LOCK_IDENTITY); + assertEquals(DeploymentConstraint.READ_ONLY, + new ManagedConfigDeploymentDetector( + installationRoot, path -> !path.equals(versionedLock), path -> true) + .detect().constraint()); + } + + @Test + void validatesExistingVersionedLockPermissionsAndIdentityWithoutMutatingIt() throws Exception { + Path config = Files.createDirectories(installationRoot.resolve("data/config")); + Path lock = config.resolve(".managed-config-v2.lock"); + SecureSetupFile.create(installationRoot, lock, new byte[0]); + assertEquals(DeploymentConstraint.UNSAFE_PATH, + new ManagedConfigDeploymentDetector(installationRoot).detect().constraint()); + + Files.delete(lock); + SecureSetupFile.create(installationRoot, lock, "malformed\n".getBytes(StandardCharsets.UTF_8)); + assertEquals(DeploymentConstraint.UNSAFE_PATH, + new ManagedConfigDeploymentDetector(installationRoot).detect().constraint()); + + Files.delete(lock); + SecureSetupFile.create(installationRoot, lock, VALID_LOCK_IDENTITY); + assertEquals(ApplyMode.MANAGED_WRITE, + new ManagedConfigDeploymentDetector(installationRoot).detect().applyMode()); + + if (Files.getFileStore(lock).supportsFileAttributeView("posix")) { + Files.setPosixFilePermissions(lock, Set.of( + PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, + PosixFilePermission.GROUP_READ, PosixFilePermission.OTHERS_READ)); + assertEquals(DeploymentConstraint.UNSAFE_PATH, + new ManagedConfigDeploymentDetector(installationRoot).detect().constraint()); + } + } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransactionTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransactionTest.java new file mode 100644 index 0000000000..2f2344e7a8 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransactionTest.java @@ -0,0 +1,493 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermission; +import java.util.Arrays; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFileLock; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class ManagedMigrationConfigurationTransactionTest { + + private static final String OPERATION = "migration-operation"; + private static final String BASE = "base-generation"; + private static final String CANDIDATE = "candidate-generation"; + private static final String IDENTITY = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + @TempDir + private Path installationRoot; + + @TempDir + private Path outsideRoot; + + @Test + void stagesCompleteOwnerOnlyCandidateWithoutChangingManagedSnapshots() throws Exception { + ManagedConfigurationTransaction setup = new ManagedConfigurationTransaction(installationRoot); + assertEquals(ManagedConfigurationTransaction.Outcome.APPLIED, setup.apply(bundle("base"))); + String actualBase = new FileManagedApplicationConfigStore(installationRoot) + .readActive().generation().orElseThrow(); + ManagedMigrationConfigurationTransaction migration = + new ManagedMigrationConfigurationTransaction(installationRoot); + + ManagedMigrationConfigurationTransaction.CandidateRef ref = + migration.stage(OPERATION, CANDIDATE, actualBase, IDENTITY, bundle("next")); + + assertEquals(new ManagedMigrationConfigurationTransaction.CandidateRef(OPERATION, CANDIDATE), ref); + assertEquals(ManagedMigrationConfigurationTransaction.CandidateState.READY, + migration.inspect(ref).state()); + assertEquals(actualBase, migration.inspect(ref).baseGeneration().orElseThrow()); + assertEquals(IDENTITY, migration.inspect(ref).targetIdentityHash().orElseThrow()); + assertEquals(configuration("base"), new FileManagedApplicationConfigStore(installationRoot) + .readActive().value().orElseThrow()); + assertEquals(CandidateState.MISSING, new FileManagedApplicationConfigStore(installationRoot) + .readCandidate().state()); + Path directory = installationRoot.resolve("data/config/migration-candidates") + .resolve(OPERATION).resolve(CANDIDATE); + try (Stream files = Files.list(directory)) { + assertEquals(Set.of("application", "manifest", "secrets"), + files.map(path -> path.getFileName().toString()).collect(java.util.stream.Collectors.toSet())); + } + if (Files.getFileStore(directory.resolve("manifest")).supportsFileAttributeView("posix")) { + for (String file : Set.of("application", "manifest", "secrets")) { + assertEquals(Set.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE), + Files.getPosixFilePermissions(directory.resolve(file))); + } + } + String applicationDocument = Files.readString(directory.resolve("application")); + String manifestDocument = Files.readString(directory.resolve("manifest")); + assertTrue(!applicationDocument.contains("database-next")); + assertTrue(!applicationDocument.contains("mail-next")); + assertTrue(!manifestDocument.contains("jdbc:postgresql")); + assertTrue(!manifestDocument.contains("telemetry-user")); + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.ALREADY_STAGED, + migration.stageOutcome(OPERATION, CANDIDATE, actualBase, IDENTITY, bundle("next"))); + } + + @Test + void rejectsStageWhenTheActivePairDoesNotMatchTheDeclaredBase() throws Exception { + new ManagedConfigurationTransaction(installationRoot).apply(bundle("base")); + ManagedMigrationConfigurationTransaction migration = + new ManagedMigrationConfigurationTransaction(installationRoot); + + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.STALE, + migration.stageOutcome(OPERATION, CANDIDATE, BASE, IDENTITY, bundle("next"))); + assertEquals(ManagedMigrationConfigurationTransaction.CandidateState.MISSING, + migration.inspect(new ManagedMigrationConfigurationTransaction.CandidateRef( + OPERATION, CANDIDATE)).state()); + } + + @Test + void readsOnlyTheExactCandidateAndClosesDecodedSecretsAfterCallback() throws Exception { + new ManagedConfigurationTransaction(installationRoot).apply(bundle("base")); + String actualBase = new FileManagedApplicationConfigStore(installationRoot) + .readActive().generation().orElseThrow(); + ManagedMigrationConfigurationTransaction migration = + new ManagedMigrationConfigurationTransaction(installationRoot); + ManagedMigrationConfigurationTransaction.CandidateRef ref = + migration.stage(OPERATION, CANDIDATE, actualBase, IDENTITY, bundle("next")); + AtomicReference observed = new AtomicReference<>(); + + assertEquals("jdbc:postgresql://db/next", migration.readExact(ref, bundle -> { + observed.set(bundle.secrets()); + assertEquals("smtp.example", bundle.application().optional().mail().orElseThrow().host()); + assertEquals("mail-next", new String(bundle.secrets().mailPassword().orElseThrow().copy())); + return bundle.application().metadataDatabase().jdbcUrl(); + })); + + assertTrue(new String(observed.get().metadataDatabasePassword().copy()).chars().allMatch(value -> value == 0)); + assertThrows(IOException.class, () -> migration.readExact( + new ManagedMigrationConfigurationTransaction.CandidateRef("wrong-operation", CANDIDATE), + candidate -> "unreachable")); + } + + @Test + void closesDecodedSecretsWhenTheSynchronousReaderFails() throws Exception { + new ManagedConfigurationTransaction(installationRoot).apply(bundle("base")); + String actualBase = new FileManagedApplicationConfigStore(installationRoot) + .readActive().generation().orElseThrow(); + ManagedMigrationConfigurationTransaction migration = + new ManagedMigrationConfigurationTransaction(installationRoot); + ManagedMigrationConfigurationTransaction.CandidateRef ref = + migration.stage(OPERATION, CANDIDATE, actualBase, IDENTITY, bundle("next")); + AtomicReference observed = new AtomicReference<>(); + + assertThrows(IllegalStateException.class, () -> migration.readExact(ref, bundle -> { + observed.set(bundle.secrets()); + Thread.currentThread().interrupt(); + throw new IllegalStateException("stop"); + })); + + assertTrue(new String(observed.get().metadataDatabasePassword().copy()).chars().allMatch(value -> value == 0)); + assertTrue(Thread.interrupted()); + } + + @Test + void exactDiscardCannotRemoveAnotherGenerationAndSetupRecoveryLeavesMigrationCandidateAlone() + throws Exception { + new ManagedConfigurationTransaction(installationRoot).apply(bundle("base")); + String actualBase = new FileManagedApplicationConfigStore(installationRoot) + .readActive().generation().orElseThrow(); + ManagedMigrationConfigurationTransaction migration = + new ManagedMigrationConfigurationTransaction(installationRoot); + ManagedMigrationConfigurationTransaction.CandidateRef ref = + migration.stage(OPERATION, CANDIDATE, actualBase, IDENTITY, bundle("next")); + + assertEquals(ManagedConfigurationTransaction.Outcome.APPLIED, + new ManagedConfigurationTransaction(installationRoot).recover()); + assertEquals(ManagedMigrationConfigurationTransaction.DiscardOutcome.NOT_FOUND, + migration.discardExact(new ManagedMigrationConfigurationTransaction.CandidateRef( + OPERATION, "later-generation"))); + assertEquals(ManagedMigrationConfigurationTransaction.CandidateState.READY, + migration.inspect(ref).state()); + assertEquals(ManagedMigrationConfigurationTransaction.DiscardOutcome.DISCARDED, + migration.discardExact(ref)); + assertEquals(ManagedMigrationConfigurationTransaction.CandidateState.MISSING, + migration.inspect(ref).state()); + } + + @Test + void partialOrTamperedCandidateFailsClosedAndCanOnlyBeDiscardedByExactReference() throws Exception { + new ManagedConfigurationTransaction(installationRoot).apply(bundle("base")); + String actualBase = new FileManagedApplicationConfigStore(installationRoot) + .readActive().generation().orElseThrow(); + ManagedMigrationConfigurationTransaction migration = + new ManagedMigrationConfigurationTransaction(installationRoot); + ManagedMigrationConfigurationTransaction.CandidateRef ref = + migration.stage(OPERATION, CANDIDATE, actualBase, IDENTITY, bundle("next")); + Path manifest = installationRoot.resolve("data/config/migration-candidates") + .resolve(OPERATION).resolve(CANDIDATE).resolve("manifest"); + Files.writeString(manifest, "tampered"); + + assertEquals(ManagedMigrationConfigurationTransaction.CandidateState.RECOVERY_REQUIRED, + migration.inspect(ref).state()); + assertThrows(IOException.class, () -> migration.readExact(ref, bundle -> "unreachable")); + assertEquals(ManagedMigrationConfigurationTransaction.DiscardOutcome.DISCARDED, + migration.discardExact(ref)); + } + + @Test + void crossDocumentCredentialMismatchFailsClosed() throws Exception { + new ManagedConfigurationTransaction(installationRoot).apply(bundle("base")); + String actualBase = new FileManagedApplicationConfigStore(installationRoot) + .readActive().generation().orElseThrow(); + ManagedMigrationConfigurationTransaction migration = + new ManagedMigrationConfigurationTransaction(installationRoot); + ManagedMigrationConfigurationTransaction.CandidateRef ref = + migration.stage(OPERATION, CANDIDATE, actualBase, IDENTITY, bundle("next")); + Path secretsFile = candidateDirectory(CANDIDATE).resolve("secrets"); + ManagedSecrets incomplete = ManagedSecrets.withoutTelemetryPassword(SecretValue.of("database-next")); + byte[] encoded = new SecretConfigDocumentCodec().encode(incomplete, CANDIDATE); + try { + Files.write(secretsFile, encoded); + } finally { + Arrays.fill(encoded, (byte) 0); + incomplete.close(); + } + + assertEquals(ManagedMigrationConfigurationTransaction.CandidateState.RECOVERY_REQUIRED, + migration.inspect(ref).state()); + assertThrows(IOException.class, () -> migration.readExact(ref, candidate -> "unreachable")); + } + + @Test + void missingSplitOrCorruptActivePairRequiresRecoveryInsteadOfReportingStale() throws Exception { + ManagedMigrationConfigurationTransaction migration = + new ManagedMigrationConfigurationTransaction(installationRoot); + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED, + migration.stageOutcome(OPERATION, CANDIDATE, BASE, IDENTITY, bundle("next"))); + + new ManagedConfigurationTransaction(installationRoot).apply(bundle("base")); + String actualBase = new FileManagedApplicationConfigStore(installationRoot) + .readActive().generation().orElseThrow(); + Path activeSecrets = installationRoot.resolve("data/config/managed-secrets.properties"); + Files.delete(activeSecrets); + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED, + migration.stageOutcome(OPERATION, CANDIDATE, actualBase, IDENTITY, bundle("next"))); + + Files.writeString(activeSecrets, "corrupt"); + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED, + migration.stageOutcome(OPERATION, CANDIDATE, actualBase, IDENTITY, bundle("next"))); + } + + @Test + void crossDocumentCredentialMismatchInTheActivePairRequiresRecovery() throws Exception { + new ManagedConfigurationTransaction(installationRoot).apply(bundle("base")); + String actualBase = new FileManagedApplicationConfigStore(installationRoot) + .readActive().generation().orElseThrow(); + ManagedSecrets incomplete = ManagedSecrets.withoutTelemetryPassword(SecretValue.of("database-base")); + byte[] encoded = new SecretConfigDocumentCodec().encode(incomplete, actualBase); + try { + Files.write(installationRoot.resolve("data/config/managed-secrets.properties"), encoded); + } finally { + Arrays.fill(encoded, (byte) 0); + incomplete.close(); + } + + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED, + new ManagedMigrationConfigurationTransaction(installationRoot).stageOutcome( + OPERATION, CANDIDATE, actualBase, IDENTITY, bundle("next"))); + } + + @Test + void finalCandidateFileSymlinksRequireRecoveryAndCannotBeDiscardedAsMissing() throws Exception { + new ManagedConfigurationTransaction(installationRoot).apply(bundle("base")); + String actualBase = new FileManagedApplicationConfigStore(installationRoot) + .readActive().generation().orElseThrow(); + ManagedMigrationConfigurationTransaction migration = + new ManagedMigrationConfigurationTransaction(installationRoot); + for (String fileName : Set.of("application", "secrets", "manifest")) { + String generation = "symlink-" + fileName; + ManagedMigrationConfigurationTransaction.CandidateRef ref = + new ManagedMigrationConfigurationTransaction.CandidateRef(OPERATION, generation); + Path candidateFile = candidateDirectory(generation).resolve(fileName); + Path outsideFile = outsideRoot.resolve(generation + "-" + fileName); + Files.createDirectories(candidateFile.getParent()); + Files.writeString(outsideFile, "outside"); + try { + Files.createSymbolicLink(candidateFile, outsideFile); + } catch (UnsupportedOperationException failure) { + return; + } + + assertEquals(ManagedMigrationConfigurationTransaction.CandidateState.RECOVERY_REQUIRED, + migration.inspect(ref).state()); + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED, + migration.stageOutcome(OPERATION, generation, actualBase, IDENTITY, bundle("next"))); + assertThrows(IOException.class, () -> migration.discardExact(ref)); + Files.delete(candidateFile); + } + } + + @Test + void everyPreManifestStageWindowIsInspectableAndExactlyDiscardable() throws Exception { + ManagedMigrationConfigurationTransaction migration = + new ManagedMigrationConfigurationTransaction(installationRoot); + for (int publishedFiles = 1; publishedFiles <= 2; publishedFiles++) { + String generation = "partial-" + publishedFiles; + ManagedMigrationConfigurationTransaction.CandidateRef ref = + new ManagedMigrationConfigurationTransaction.CandidateRef(OPERATION, generation); + Path directory = installationRoot.resolve("data/config/migration-candidates") + .resolve(OPERATION).resolve(generation); + byte[] application = new ApplicationConfigDocumentCodec().encode( + configuration("partial"), generation); + ManagedSecrets sourceSecrets = new ManagedSecrets(SecretValue.of("database-partial"), + java.util.Optional.of(SecretValue.of("telemetry-partial")), + java.util.Optional.of(SecretValue.of("mail-partial"))); + byte[] secrets = new SecretConfigDocumentCodec().encode(sourceSecrets, generation); + try { + SecureSetupFile.create(installationRoot, directory.resolve("application"), application); + if (publishedFiles == 2) { + SecureSetupFile.create(installationRoot, directory.resolve("secrets"), secrets); + } + } finally { + Arrays.fill(application, (byte) 0); + Arrays.fill(secrets, (byte) 0); + sourceSecrets.close(); + } + + assertEquals(ManagedMigrationConfigurationTransaction.CandidateState.RECOVERY_REQUIRED, + migration.inspect(ref).state()); + assertEquals(ManagedMigrationConfigurationTransaction.DiscardOutcome.DISCARDED, + migration.discardExact(ref)); + assertEquals(ManagedMigrationConfigurationTransaction.CandidateState.MISSING, + migration.inspect(ref).state()); + } + } + + @Test + void candidateReferencesAndOutcomesDoNotExposeIdentityOrPaths() { + ManagedMigrationConfigurationTransaction.CandidateRef ref = + new ManagedMigrationConfigurationTransaction.CandidateRef(OPERATION, CANDIDATE); + + assertEquals("CandidateRef[operationId=migration-operation, candidateGeneration=candidate-generation]", + ref.toString()); + assertTrue(Stream.of(ManagedMigrationConfigurationTransaction.StageOutcome.values()) + .map(Enum::name).noneMatch(value -> value.contains("/"))); + assertEquals("Inspection[state=READY, exactMetadata=true]", + new ManagedMigrationConfigurationTransaction.Inspection( + ManagedMigrationConfigurationTransaction.CandidateState.READY, + java.util.Optional.of(BASE), java.util.Optional.of(IDENTITY)).toString()); + } + + @Test + void rejectsTraversalAndSentinelIdentifiersBeforeTouchingTheFilesystem() { + assertThrows(IllegalArgumentException.class, + () -> new ManagedMigrationConfigurationTransaction.CandidateRef("../escape", CANDIDATE)); + assertThrows(IllegalArgumentException.class, + () -> new ManagedMigrationConfigurationTransaction.CandidateRef(OPERATION, "-")); + assertThrows(IllegalArgumentException.class, + () -> new ManagedMigrationConfigurationTransaction.CandidateRef(OPERATION, "../escape")); + assertEquals("a.b_c-d", new ManagedMigrationConfigurationTransaction.CandidateRef( + "a.b_c-d", CANDIDATE).operationId()); + assertEquals(128, new ManagedMigrationConfigurationTransaction.CandidateRef( + "a" + "b".repeat(127), CANDIDATE).operationId().length()); + } + + @Test + void symlinkedCandidateDirectoryFailsClosedWithoutWritingOutsideTheRoot() throws Exception { + new ManagedConfigurationTransaction(installationRoot).apply(bundle("base")); + String actualBase = new FileManagedApplicationConfigStore(installationRoot) + .readActive().generation().orElseThrow(); + Path outside = Files.createDirectories(outsideRoot.resolve("outside-candidates")); + Path candidateRoot = Files.createDirectories( + installationRoot.resolve("data/config/migration-candidates")); + try { + Files.createSymbolicLink(candidateRoot.resolve(OPERATION), outside); + } catch (UnsupportedOperationException failure) { + return; + } + ManagedMigrationConfigurationTransaction migration = + new ManagedMigrationConfigurationTransaction(installationRoot); + + assertThrows(IOException.class, () -> migration.stage( + OPERATION, CANDIDATE, actualBase, IDENTITY, bundle("next"))); + try (Stream files = Files.list(outside)) { + assertEquals(0, files.count()); + } + } + + @Test + void decodedMaterialAlwaysClearsItsOwnedSecrets() { + ManagedSecrets decoded = new ManagedSecrets(SecretValue.of("database-owned"), + java.util.Optional.of(SecretValue.of("telemetry-owned")), + java.util.Optional.of(SecretValue.of("mail-owned"))); + MigrationCandidateManifest manifest = new MigrationCandidateManifest( + OPERATION, CANDIDATE, BASE, IDENTITY); + + try (MigrationCandidateMaterial ignored = MigrationCandidateMaterial.ready( + manifest, configuration("owned"), decoded)) { + assertEquals("database-owned", new String(decoded.metadataDatabasePassword().copy())); + } + + assertTrue(new String(decoded.metadataDatabasePassword().copy()).chars().allMatch(value -> value == 0)); + assertTrue(new String(decoded.telemetryPassword().orElseThrow().copy()).chars().allMatch(value -> value == 0)); + assertTrue(new String(decoded.mailPassword().orElseThrow().copy()).chars().allMatch(value -> value == 0)); + } + + @Test + void setupAndMigrationShareTheSameSecureLockInode() throws Exception { + ManagedConfigurationTransaction setup = new ManagedConfigurationTransaction(installationRoot); + assertEquals(ManagedConfigurationTransaction.Outcome.APPLIED, setup.apply(bundle("base"))); + String actualBase = new FileManagedApplicationConfigStore(installationRoot) + .readActive().generation().orElseThrow(); + ManagedMigrationConfigurationTransaction migration = + new ManagedMigrationConfigurationTransaction(installationRoot); + SecureSetupFileLock held = new SecureSetupFileLock( + installationRoot, "data/config/.managed-config-v2.lock"); + CountDownLatch locked = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch callersStarted = new CountDownLatch(2); + ExecutorService executor = Executors.newFixedThreadPool(3); + try { + Future holder = executor.submit(() -> { + held.execute(() -> { + locked.countDown(); + try { + release.await(); + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + throw new IOException("Lock holder interrupted", failure); + } + }); + return null; + }); + locked.await(); + Future setupWriter = executor.submit(() -> { + callersStarted.countDown(); + return setup.apply(bundle("setup-next")); + }); + Future migrationWriter = executor.submit(() -> { + callersStarted.countDown(); + return migration.stageOutcome( + OPERATION, CANDIDATE, actualBase, IDENTITY, bundle("migration-next")); + }); + callersStarted.await(); + + assertTrue(!setupWriter.isDone()); + assertTrue(!migrationWriter.isDone()); + release.countDown(); + holder.get(); + setupWriter.get(); + migrationWriter.get(); + } finally { + release.countDown(); + executor.shutdownNow(); + } + } + + @Test + void legacyUnidentifiedLockFileDoesNotBlockTheVersionedSharedLock() throws Exception { + Path configDirectory = Files.createDirectories(installationRoot.resolve("data/config")); + Path legacyLock = configDirectory.resolve(".managed-config.lock"); + Files.write(legacyLock, new byte[0]); + if (Files.getFileStore(legacyLock).supportsFileAttributeView("posix")) { + Files.setPosixFilePermissions(legacyLock, Set.of( + PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, + PosixFilePermission.GROUP_READ, PosixFilePermission.OTHERS_READ)); + } + + ManagedConfigurationTransaction setup = new ManagedConfigurationTransaction(installationRoot); + assertEquals(ManagedConfigurationTransaction.Outcome.APPLIED, setup.apply(bundle("base"))); + String actualBase = new FileManagedApplicationConfigStore(installationRoot) + .readActive().generation().orElseThrow(); + ManagedMigrationConfigurationTransaction migration = + new ManagedMigrationConfigurationTransaction(installationRoot); + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.STAGED, + migration.stageOutcome(OPERATION, CANDIDATE, actualBase, IDENTITY, bundle("next"))); + + assertEquals(0, Files.size(legacyLock)); + assertTrue(SecureSetupFile.isOwnerOnlyRegularFile( + configDirectory.resolve(".managed-config-v2.lock"))); + } + + private Path candidateDirectory(String generation) { + return installationRoot.resolve("data/config/migration-candidates") + .resolve(OPERATION).resolve(generation); + } + + private static ManagedConfigurationBundle bundle(String suffix) { + return new ManagedConfigurationBundle(configuration(suffix), + new ManagedSecrets(SecretValue.of("database-" + suffix), + java.util.Optional.of(SecretValue.of("telemetry-" + suffix)), + java.util.Optional.of(SecretValue.of("mail-" + suffix)))); + } + + private static ManagedApplicationConfig configuration(String suffix) { + return new ManagedApplicationConfig( + new MetadataDatabaseSettings(MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://db/" + suffix, "hertzbeat"), + new GreptimeSettings(new GreptimeEndpoints("greptime:4001", "http://greptime:4000"), + "public", java.util.Optional.of("telemetry-user")), + optionalConfiguration()); + } + + private static ManagedOptionalConfiguration optionalConfiguration() { + ManagedOptionalConfiguration.MailSettings mail = new ManagedOptionalConfiguration.MailSettings( + "smtp.example", 587, org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity.STARTTLS, + java.util.Optional.of("mailer"), "alerts@example.org"); + return new ManagedOptionalConfiguration(java.util.Optional.empty(), java.util.Optional.empty(), + java.util.Optional.of(mail)); + } +} From 7afb6501e7712342c45ef45abd7220fec142705b Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 07:25:12 +0800 Subject: [PATCH 43/71] Activate managed migration candidates --- .../setup/config/ExactSnapshotOutcome.java | 16 + .../FileManagedApplicationConfigStore.java | 24 + .../setup/config/FileManagedSecretStore.java | 22 + .../config/FileManagedSnapshotStore.java | 120 +++++ .../manager/setup/config/ManagedFileIo.java | 4 + .../config/ManagedMigrationActivation.java | 76 +++ ...agedMigrationConfigurationTransaction.java | 32 ++ .../config/MigrationActivationCandidate.java | 25 + .../config/MigrationActivationClassifier.java | 196 +++++++ .../config/MigrationActivationSnapshots.java | 34 ++ .../MigrationActivationStepExecutor.java | 74 +++ .../setup/config/MigrationCandidateStore.java | 12 + .../setup/config/NioManagedFilePublisher.java | 18 +- ...CommittedSetupFileDurabilityException.java | 2 +- .../FileManagedConfigurationStoreTest.java | 5 + .../FileManagedSnapshotStoreBufferTest.java | 5 + .../ManagedConfigurationTransactionTest.java | 10 + .../ManagedMigrationActivationTest.java | 509 ++++++++++++++++++ .../MigrationActivationClassifierTest.java | 114 ++++ 19 files changed, 1295 insertions(+), 3 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ExactSnapshotOutcome.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationActivation.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationActivationCandidate.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationActivationClassifier.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationActivationSnapshots.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationActivationStepExecutor.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationActivationTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/MigrationActivationClassifierTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ExactSnapshotOutcome.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ExactSnapshotOutcome.java new file mode 100644 index 0000000000..ea2ee1d124 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ExactSnapshotOutcome.java @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +/** Internal result of one exact, generation-bound snapshot mutation. */ +enum ExactSnapshotOutcome { + APPLIED, + ALREADY_APPLIED, + STALE, + RECOVERY_REQUIRED +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedApplicationConfigStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedApplicationConfigStore.java index 4df48d6a47..3ec4bbd991 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedApplicationConfigStore.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedApplicationConfigStore.java @@ -76,4 +76,28 @@ final class FileManagedApplicationConfigStore implements ManagedApplicationConfi public void discardCandidate() throws IOException { delegate.discardCandidate(); } + + ExactSnapshotOutcome stageCandidateExact(ManagedApplicationConfig candidate, String generation) + throws IOException { + return delegate.stageCandidateExact(candidate, generation); + } + + ExactSnapshotOutcome promoteCandidateExact(ManagedApplicationConfig candidate, String generation, + String baseGeneration) throws IOException { + return delegate.promoteCandidateExact(candidate, generation, baseGeneration); + } + + ExactSnapshotOutcome restoreActiveExact(ManagedApplicationConfig candidate, String generation, + String baseGeneration) throws IOException { + return delegate.restoreActiveExact(candidate, generation, baseGeneration); + } + + ExactSnapshotOutcome discardCandidateExact(ManagedApplicationConfig candidate, String generation) + throws IOException { + return delegate.discardCandidateExact(candidate, generation); + } + + void confirmDurability() throws IOException { + delegate.confirmDurability(); + } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedSecretStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedSecretStore.java index 4a1cc043c1..82abcb818a 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedSecretStore.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedSecretStore.java @@ -75,4 +75,26 @@ final class FileManagedSecretStore implements ManagedSecretStore { public void discardCandidate() throws IOException { delegate.discardCandidate(); } + + ExactSnapshotOutcome stageCandidateExact(ManagedSecrets candidate, String generation) throws IOException { + return delegate.stageCandidateExact(candidate, generation); + } + + ExactSnapshotOutcome promoteCandidateExact(ManagedSecrets candidate, String generation, + String baseGeneration) throws IOException { + return delegate.promoteCandidateExact(candidate, generation, baseGeneration); + } + + ExactSnapshotOutcome restoreActiveExact(ManagedSecrets candidate, String generation, + String baseGeneration) throws IOException { + return delegate.restoreActiveExact(candidate, generation, baseGeneration); + } + + ExactSnapshotOutcome discardCandidateExact(ManagedSecrets candidate, String generation) throws IOException { + return delegate.discardCandidateExact(candidate, generation); + } + + void confirmDurability() throws IOException { + delegate.confirmDurability(); + } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedSnapshotStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedSnapshotStore.java index 82f7fb10ef..14aecc685a 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedSnapshotStore.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/FileManagedSnapshotStore.java @@ -127,6 +127,109 @@ final class FileManagedSnapshotStore { publisher.remove(candidate); } + ExactSnapshotOutcome stageCandidateExact(T expected, String generation) throws IOException { + ensureSafePaths(); + CandidateRead current = readCandidate(); + try { + if (current.state() == CandidateState.MISSING) { + publishEncoded(candidate, expected, generation); + return ExactSnapshotOutcome.APPLIED; + } + return matches(current, expected, generation) + ? ExactSnapshotOutcome.ALREADY_APPLIED : ExactSnapshotOutcome.RECOVERY_REQUIRED; + } finally { + closeRead(current); + } + } + + ExactSnapshotOutcome promoteCandidateExact(T expected, String generation, + String baseGeneration) throws IOException { + ensureSafePaths(); + CandidateRead activeRead = readActive(); + CandidateRead candidateRead = readCandidate(); + CandidateRead lastKnownGoodRead = readLastKnownGood(); + try { + if (matches(activeRead, expected, generation)) { + if (!hasGeneration(lastKnownGoodRead, baseGeneration) + || !(candidateRead.state() == CandidateState.MISSING + || matches(candidateRead, expected, generation))) { + return ExactSnapshotOutcome.RECOVERY_REQUIRED; + } + if (candidateRead.state() == CandidateState.VALID) { + publisher.remove(candidate); + } + return ExactSnapshotOutcome.ALREADY_APPLIED; + } + if (activeRead.state() == CandidateState.VALID + && !hasGeneration(activeRead, baseGeneration)) { + return ExactSnapshotOutcome.STALE; + } + if (!hasGeneration(activeRead, baseGeneration) + || !matches(candidateRead, expected, generation)) { + return ExactSnapshotOutcome.RECOVERY_REQUIRED; + } + if (!sameSnapshot(activeRead, lastKnownGoodRead)) { + publishEncoded(lastKnownGood, activeRead.value().orElseThrow(), baseGeneration); + } + publishEncoded(active, expected, generation); + publisher.remove(candidate); + return ExactSnapshotOutcome.APPLIED; + } finally { + closeRead(activeRead); + closeRead(candidateRead); + closeRead(lastKnownGoodRead); + } + } + + ExactSnapshotOutcome restoreActiveExact(T expectedTarget, String targetGeneration, + String baseGeneration) throws IOException { + ensureSafePaths(); + CandidateRead activeRead = readActive(); + CandidateRead lastKnownGoodRead = readLastKnownGood(); + try { + if (!hasGeneration(lastKnownGoodRead, baseGeneration)) { + return ExactSnapshotOutcome.RECOVERY_REQUIRED; + } + if (sameSnapshot(activeRead, lastKnownGoodRead)) { + return ExactSnapshotOutcome.ALREADY_APPLIED; + } + if (activeRead.state() == CandidateState.VALID + && !matches(activeRead, expectedTarget, targetGeneration)) { + return ExactSnapshotOutcome.STALE; + } + if (!matches(activeRead, expectedTarget, targetGeneration)) { + return ExactSnapshotOutcome.RECOVERY_REQUIRED; + } + publishEncoded(active, lastKnownGoodRead.value().orElseThrow(), baseGeneration); + return ExactSnapshotOutcome.APPLIED; + } finally { + closeRead(activeRead); + closeRead(lastKnownGoodRead); + } + } + + ExactSnapshotOutcome discardCandidateExact(T expected, String generation) throws IOException { + ensureSafePaths(); + CandidateRead current = readCandidate(); + try { + if (current.state() == CandidateState.MISSING) { + return ExactSnapshotOutcome.ALREADY_APPLIED; + } + if (!matches(current, expected, generation)) { + return ExactSnapshotOutcome.RECOVERY_REQUIRED; + } + publisher.remove(candidate); + return ExactSnapshotOutcome.APPLIED; + } finally { + closeRead(current); + } + } + + void confirmDurability() throws IOException { + ensureSafePaths(); + publisher.confirmDurability(active); + } + private CandidateRead read(Path path) { if (isUnsafePath(path)) { return CandidateRead.unreadable(); @@ -181,6 +284,23 @@ final class FileManagedSnapshotStore { } } + private static boolean hasGeneration(CandidateRead read, String generation) { + return read.state() == CandidateState.VALID && read.generation().filter(generation::equals).isPresent(); + } + + private static boolean matches(CandidateRead read, T expected, String generation) { + return hasGeneration(read, generation) && read.value().filter(expected::equals).isPresent(); + } + + private static boolean sameSnapshot(CandidateRead left, CandidateRead right) { + return left.state() == CandidateState.VALID && right.state() == CandidateState.VALID + && left.generation().equals(right.generation()) && left.value().equals(right.value()); + } + + private static void closeRead(CandidateRead read) { + read.value().ifPresent(FileManagedSnapshotStore::close); + } + private static void clear(byte[] content) { if (content != null) { Arrays.fill(content, (byte) 0); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedFileIo.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedFileIo.java index 61c60281cc..ad33b79b10 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedFileIo.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedFileIo.java @@ -32,6 +32,9 @@ final class ManagedFileIo { void publish(Path target, byte[] content, boolean ownerOnly) throws IOException; void remove(Path target) throws IOException; + + /** Confirms that prior directory-entry mutations for the target are durably published. */ + void confirmDurability(Path target) throws IOException; } @FunctionalInterface @@ -47,4 +50,5 @@ final class ManagedFileIo { void forceDirectory(Path directory) throws IOException; } + } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationActivation.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationActivation.java new file mode 100644 index 0000000000..f46fc00083 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationActivation.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.io.IOException; + +/** Reconciles one exact migration candidate without owning snapshot classification or file I/O. */ +final class ManagedMigrationActivation { + + private static final int MAXIMUM_RECONCILIATION_STEPS = 16; + + private final MigrationActivationClassifier classifier = new MigrationActivationClassifier(); + private final MigrationActivationStepExecutor executor; + + ManagedMigrationActivation(FileManagedApplicationConfigStore applications, + FileManagedSecretStore secrets) { + executor = new MigrationActivationStepExecutor(applications, secrets); + } + + ManagedMigrationConfigurationTransaction.ActivationOutcome activate(MigrationCandidateMaterial material) { + MigrationActivationCandidate candidate = MigrationActivationCandidate.from(material); + boolean changed = false; + for (int step = 0; step < MAXIMUM_RECONCILIATION_STEPS; step++) { + try (MigrationActivationSnapshots snapshots = executor.readSnapshots()) { + MigrationActivationClassifier.Decision decision = classifier.activation(candidate, snapshots); + if (decision == MigrationActivationClassifier.Decision.COMPLETE) { + executor.confirmDurability(); + return changed ? ManagedMigrationConfigurationTransaction.ActivationOutcome.ACTIVATED + : ManagedMigrationConfigurationTransaction.ActivationOutcome.ALREADY_ACTIVE; + } + if (decision == MigrationActivationClassifier.Decision.STALE) { + return ManagedMigrationConfigurationTransaction.ActivationOutcome.STALE; + } + if (decision == MigrationActivationClassifier.Decision.RECOVERY_REQUIRED) { + return ManagedMigrationConfigurationTransaction.ActivationOutcome.RECOVERY_REQUIRED; + } + changed = true; + executor.activate(decision, candidate); + } catch (IOException ignored) { + return ManagedMigrationConfigurationTransaction.ActivationOutcome.RECOVERY_REQUIRED; + } + } + return ManagedMigrationConfigurationTransaction.ActivationOutcome.RECOVERY_REQUIRED; + } + + ManagedMigrationConfigurationTransaction.RollbackOutcome rollback(MigrationCandidateMaterial material) { + MigrationActivationCandidate candidate = MigrationActivationCandidate.from(material); + boolean changed = false; + for (int step = 0; step < MAXIMUM_RECONCILIATION_STEPS; step++) { + try (MigrationActivationSnapshots snapshots = executor.readSnapshots()) { + MigrationActivationClassifier.Decision decision = classifier.rollback(candidate, snapshots); + if (decision == MigrationActivationClassifier.Decision.COMPLETE) { + executor.confirmDurability(); + return changed ? ManagedMigrationConfigurationTransaction.RollbackOutcome.ROLLED_BACK + : ManagedMigrationConfigurationTransaction.RollbackOutcome.ALREADY_ROLLED_BACK; + } + if (decision == MigrationActivationClassifier.Decision.STALE) { + return ManagedMigrationConfigurationTransaction.RollbackOutcome.STALE; + } + if (decision == MigrationActivationClassifier.Decision.RECOVERY_REQUIRED) { + return ManagedMigrationConfigurationTransaction.RollbackOutcome.RECOVERY_REQUIRED; + } + changed = true; + executor.rollback(decision, candidate); + } catch (IOException ignored) { + return ManagedMigrationConfigurationTransaction.RollbackOutcome.RECOVERY_REQUIRED; + } + } + return ManagedMigrationConfigurationTransaction.RollbackOutcome.RECOVERY_REQUIRED; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransaction.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransaction.java index c2323a84d5..42da189739 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransaction.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransaction.java @@ -22,11 +22,15 @@ public final class ManagedMigrationConfigurationTransaction { private final ManagedConfigurationLock lock; private final MigrationCandidateStore store; + private final ManagedMigrationActivation activation; /** Creates the production migration candidate transaction. */ public ManagedMigrationConfigurationTransaction(Path installationRoot) { lock = new ManagedConfigurationLock(installationRoot); store = new MigrationCandidateStore(installationRoot); + activation = new ManagedMigrationActivation( + new FileManagedApplicationConfigStore(installationRoot), + new FileManagedSecretStore(installationRoot)); } /** Stages the exact candidate or fails with a stable, secret-free error. */ @@ -70,6 +74,28 @@ public final class ManagedMigrationConfigurationTransaction { return lock.execute(() -> store.discardExact(reference)); } + /** Activates only the exact candidate over its recorded base generation. */ + public ActivationOutcome activate(CandidateRef reference) throws IOException { + Objects.requireNonNull(reference, "reference"); + return lock.execute(() -> store.withMaterial(reference, material -> { + if (material.inspection().state() != CandidateState.READY) { + return ActivationOutcome.RECOVERY_REQUIRED; + } + return activation.activate(material); + })); + } + + /** Restores only the exact recorded base while the candidate generation remains active. */ + public RollbackOutcome rollback(CandidateRef reference) throws IOException { + Objects.requireNonNull(reference, "reference"); + return lock.execute(() -> store.withMaterial(reference, material -> { + if (material.inspection().state() != CandidateState.READY) { + return RollbackOutcome.RECOVERY_REQUIRED; + } + return activation.rollback(material); + })); + } + static void requireGeneration(String value, String label) { Objects.requireNonNull(value, label); if (!GENERATION.matcher(value).matches()) { @@ -122,6 +148,12 @@ public final class ManagedMigrationConfigurationTransaction { /** Stable exact-discard result. */ public enum DiscardOutcome { DISCARDED, NOT_FOUND } + /** Stable exact activation result. */ + public enum ActivationOutcome { ACTIVATED, ALREADY_ACTIVE, STALE, RECOVERY_REQUIRED } + + /** Stable exact rollback result. */ + public enum RollbackOutcome { ROLLED_BACK, ALREADY_ROLLED_BACK, STALE, RECOVERY_REQUIRED } + /** Synchronous, non-retaining access to a decoded candidate bundle. */ @FunctionalInterface public interface CandidateReader { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationActivationCandidate.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationActivationCandidate.java new file mode 100644 index 0000000000..a674578f7a --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationActivationCandidate.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +/** Exact non-persistent activation input derived from one validated migration candidate. */ +record MigrationActivationCandidate(String generation, String baseGeneration, + ManagedApplicationConfig application, ManagedSecrets secrets) { + + static MigrationActivationCandidate from(MigrationCandidateMaterial material) { + MigrationCandidateManifest manifest = material.manifest().orElseThrow(); + return new MigrationActivationCandidate(manifest.candidateGeneration(), manifest.baseGeneration(), + material.application().orElseThrow(), material.secrets().orElseThrow()); + } + + @Override + public String toString() { + return "MigrationActivationCandidate[generation=" + generation + + ", baseGeneration=" + baseGeneration + ", configuration=]"; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationActivationClassifier.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationActivationClassifier.java new file mode 100644 index 0000000000..72a6127b09 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationActivationClassifier.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +/** Pure classifier for the exact activation and rollback snapshot state graph. */ +final class MigrationActivationClassifier { + + Decision activation(MigrationActivationCandidate candidate, MigrationActivationSnapshots snapshots) { + MemberState application = memberState(snapshots.activeApplication(), candidate.application(), candidate); + MemberState secret = memberState(snapshots.activeSecrets(), candidate.secrets(), candidate); + SetupCandidateState applicationCandidate = candidateState( + snapshots.candidateApplication(), candidate.application(), candidate.generation()); + SetupCandidateState secretCandidate = candidateState( + snapshots.candidateSecrets(), candidate.secrets(), candidate.generation()); + Decision invalid = invalidDecision( + snapshots, application, secret, applicationCandidate, secretCandidate); + if (invalid != null) { + return invalid; + } + if (!validBasePair(candidate, snapshots, application, secret)) { + return Decision.RECOVERY_REQUIRED; + } + if (application == MemberState.BASE && secret == MemberState.BASE) { + return bothBaseActivation(applicationCandidate, secretCandidate); + } + if (application == MemberState.TARGET && secret == MemberState.BASE) { + if (applicationCandidate == SetupCandidateState.TARGET) { + return Decision.DISCARD_APPLICATION; + } + return secretCandidate == SetupCandidateState.TARGET + ? Decision.PROMOTE_SECRET : Decision.RECOVERY_REQUIRED; + } + if (application == MemberState.BASE && secret == MemberState.TARGET) { + return Decision.RECOVERY_REQUIRED; + } + if (applicationCandidate == SetupCandidateState.TARGET) { + return Decision.DISCARD_APPLICATION; + } + if (secretCandidate == SetupCandidateState.TARGET) { + return Decision.DISCARD_SECRET; + } + return Decision.COMPLETE; + } + + Decision rollback(MigrationActivationCandidate candidate, MigrationActivationSnapshots snapshots) { + MemberState application = memberState(snapshots.activeApplication(), candidate.application(), candidate); + MemberState secret = memberState(snapshots.activeSecrets(), candidate.secrets(), candidate); + SetupCandidateState applicationCandidate = candidateState( + snapshots.candidateApplication(), candidate.application(), candidate.generation()); + SetupCandidateState secretCandidate = candidateState( + snapshots.candidateSecrets(), candidate.secrets(), candidate.generation()); + Decision invalid = invalidDecision( + snapshots, application, secret, applicationCandidate, secretCandidate); + if (invalid != null) { + return invalid; + } + if (!validLastKnownGood(candidate, snapshots)) { + return Decision.RECOVERY_REQUIRED; + } + if (application == MemberState.TARGET && secret == MemberState.TARGET) { + return Decision.RESTORE_SECRET; + } + if (application == MemberState.TARGET && secret == MemberState.BASE) { + return sameSnapshot(snapshots.activeSecrets(), snapshots.lastKnownGoodSecrets()) + ? Decision.RESTORE_APPLICATION : Decision.RECOVERY_REQUIRED; + } + if (application == MemberState.BASE && secret == MemberState.TARGET) { + return Decision.RECOVERY_REQUIRED; + } + if (!sameSnapshot(snapshots.activeApplication(), snapshots.lastKnownGoodApplication()) + || !sameSnapshot(snapshots.activeSecrets(), snapshots.lastKnownGoodSecrets())) { + return Decision.RECOVERY_REQUIRED; + } + if (applicationCandidate == SetupCandidateState.TARGET) { + return Decision.DISCARD_APPLICATION; + } + if (secretCandidate == SetupCandidateState.TARGET) { + return Decision.DISCARD_SECRET; + } + return Decision.COMPLETE; + } + + private static Decision bothBaseActivation(SetupCandidateState application, + SetupCandidateState secrets) { + if (application == SetupCandidateState.MISSING && secrets == SetupCandidateState.MISSING) { + return Decision.STAGE_APPLICATION; + } + if (application == SetupCandidateState.TARGET && secrets == SetupCandidateState.MISSING) { + return Decision.STAGE_SECRET; + } + if (application == SetupCandidateState.TARGET && secrets == SetupCandidateState.TARGET) { + return Decision.PROMOTE_APPLICATION; + } + return Decision.RECOVERY_REQUIRED; + } + + private static Decision invalidDecision(MigrationActivationSnapshots snapshots, + MemberState application, MemberState secrets, + SetupCandidateState applicationCandidate, + SetupCandidateState secretCandidate) { + if (application == MemberState.STALE || secrets == MemberState.STALE) { + return application == MemberState.STALE && secrets == MemberState.STALE + && validPair(snapshots.activeApplication(), snapshots.activeSecrets(), + snapshots.activeApplication().generation().orElseThrow()) + ? Decision.STALE : Decision.RECOVERY_REQUIRED; + } + if (application == MemberState.INVALID || secrets == MemberState.INVALID + || applicationCandidate == SetupCandidateState.INVALID + || secretCandidate == SetupCandidateState.INVALID) { + return Decision.RECOVERY_REQUIRED; + } + return null; + } + + private static boolean validBasePair(MigrationActivationCandidate candidate, + MigrationActivationSnapshots snapshots, + MemberState application, MemberState secrets) { + if (application == MemberState.BASE && secrets == MemberState.BASE) { + return validPair(snapshots.activeApplication(), snapshots.activeSecrets(), candidate.baseGeneration()); + } + if (application == MemberState.TARGET && secrets == MemberState.BASE) { + return validPair(snapshots.lastKnownGoodApplication(), snapshots.activeSecrets(), + candidate.baseGeneration()); + } + if (application == MemberState.BASE && secrets == MemberState.TARGET) { + return validPair(snapshots.activeApplication(), snapshots.lastKnownGoodSecrets(), + candidate.baseGeneration()); + } + return validLastKnownGood(candidate, snapshots); + } + + private static boolean validLastKnownGood(MigrationActivationCandidate candidate, + MigrationActivationSnapshots snapshots) { + return validPair(snapshots.lastKnownGoodApplication(), snapshots.lastKnownGoodSecrets(), + candidate.baseGeneration()); + } + + private static boolean validPair(CandidateRead application, + CandidateRead secrets, String generation) { + if (!hasGeneration(application, generation) || !hasGeneration(secrets, generation)) { + return false; + } + try { + new ManagedConfigurationBundle(application.value().orElseThrow(), secrets.value().orElseThrow()); + return true; + } catch (IllegalArgumentException failure) { + return false; + } + } + + private static MemberState memberState(CandidateRead active, T target, + MigrationActivationCandidate candidate) { + if (matches(active, target, candidate.generation())) { + return MemberState.TARGET; + } + if (hasGeneration(active, candidate.baseGeneration())) { + return MemberState.BASE; + } + return active.state() == CandidateState.VALID ? MemberState.STALE : MemberState.INVALID; + } + + private static SetupCandidateState candidateState(CandidateRead read, T target, String generation) { + if (read.state() == CandidateState.MISSING) { + return SetupCandidateState.MISSING; + } + return matches(read, target, generation) ? SetupCandidateState.TARGET : SetupCandidateState.INVALID; + } + + private static boolean hasGeneration(CandidateRead read, String generation) { + return read.state() == CandidateState.VALID && read.generation().filter(generation::equals).isPresent(); + } + + private static boolean matches(CandidateRead read, T expected, String generation) { + return hasGeneration(read, generation) && read.value().filter(expected::equals).isPresent(); + } + + private static boolean sameSnapshot(CandidateRead left, CandidateRead right) { + return left.state() == CandidateState.VALID && right.state() == CandidateState.VALID + && left.generation().equals(right.generation()) && left.value().equals(right.value()); + } + + enum Decision { + STAGE_APPLICATION, STAGE_SECRET, PROMOTE_APPLICATION, PROMOTE_SECRET, + RESTORE_APPLICATION, RESTORE_SECRET, DISCARD_APPLICATION, DISCARD_SECRET, + COMPLETE, STALE, RECOVERY_REQUIRED + } + + private enum MemberState { BASE, TARGET, STALE, INVALID } + + private enum SetupCandidateState { MISSING, TARGET, INVALID } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationActivationSnapshots.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationActivationSnapshots.java new file mode 100644 index 0000000000..4b175bc356 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationActivationSnapshots.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +/** One closeable authoritative read of all snapshots used by the activation classifier. */ +record MigrationActivationSnapshots( + CandidateRead activeApplication, + CandidateRead candidateApplication, + CandidateRead lastKnownGoodApplication, + CandidateRead activeSecrets, + CandidateRead candidateSecrets, + CandidateRead lastKnownGoodSecrets) implements AutoCloseable { + + @Override + public void close() { + ManagedConfigurationTransaction.close(activeSecrets); + ManagedConfigurationTransaction.close(candidateSecrets); + ManagedConfigurationTransaction.close(lastKnownGoodSecrets); + } + + @Override + public String toString() { + return "MigrationActivationSnapshots[applicationStates=" + + activeApplication.state() + "/" + candidateApplication.state() + "/" + + lastKnownGoodApplication.state() + ", secretStates=" + + activeSecrets.state() + "/" + candidateSecrets.state() + "/" + + lastKnownGoodSecrets.state() + "]"; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationActivationStepExecutor.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationActivationStepExecutor.java new file mode 100644 index 0000000000..40858ae799 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationActivationStepExecutor.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.io.IOException; +import java.util.Objects; + +/** Executes one exact snapshot mutation selected from an authoritative read. */ +final class MigrationActivationStepExecutor { + + private final FileManagedApplicationConfigStore applications; + private final FileManagedSecretStore secrets; + + MigrationActivationStepExecutor(FileManagedApplicationConfigStore applications, + FileManagedSecretStore secrets) { + this.applications = Objects.requireNonNull(applications, "applications"); + this.secrets = Objects.requireNonNull(secrets, "secrets"); + } + + MigrationActivationSnapshots readSnapshots() { + return new MigrationActivationSnapshots(applications.readActive(), applications.readCandidate(), + applications.readLastKnownGood(), secrets.readActive(), secrets.readCandidate(), + secrets.readLastKnownGood()); + } + + void confirmDurability() throws IOException { + applications.confirmDurability(); + secrets.confirmDurability(); + } + + void activate(MigrationActivationClassifier.Decision decision, + MigrationActivationCandidate candidate) throws IOException { + ExactSnapshotOutcome outcome = switch (decision) { + case STAGE_APPLICATION -> applications.stageCandidateExact( + candidate.application(), candidate.generation()); + case STAGE_SECRET -> secrets.stageCandidateExact(candidate.secrets(), candidate.generation()); + case PROMOTE_APPLICATION -> applications.promoteCandidateExact( + candidate.application(), candidate.generation(), candidate.baseGeneration()); + case PROMOTE_SECRET -> secrets.promoteCandidateExact( + candidate.secrets(), candidate.generation(), candidate.baseGeneration()); + case DISCARD_APPLICATION -> applications.discardCandidateExact( + candidate.application(), candidate.generation()); + case DISCARD_SECRET -> secrets.discardCandidateExact(candidate.secrets(), candidate.generation()); + default -> throw new IllegalStateException("Not an activation step"); + }; + requireApplied(outcome); + } + + void rollback(MigrationActivationClassifier.Decision decision, + MigrationActivationCandidate candidate) throws IOException { + ExactSnapshotOutcome outcome = switch (decision) { + case RESTORE_SECRET -> secrets.restoreActiveExact( + candidate.secrets(), candidate.generation(), candidate.baseGeneration()); + case RESTORE_APPLICATION -> applications.restoreActiveExact( + candidate.application(), candidate.generation(), candidate.baseGeneration()); + case DISCARD_APPLICATION -> applications.discardCandidateExact( + candidate.application(), candidate.generation()); + case DISCARD_SECRET -> secrets.discardCandidateExact(candidate.secrets(), candidate.generation()); + default -> throw new IllegalStateException("Not a rollback step"); + }; + requireApplied(outcome); + } + + static void requireApplied(ExactSnapshotOutcome outcome) throws IOException { + if (outcome != ExactSnapshotOutcome.APPLIED && outcome != ExactSnapshotOutcome.ALREADY_APPLIED) { + throw new IOException("Exact managed snapshot mutation was not applied"); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateStore.java index 5cf2b08e99..2461568deb 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateStore.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateStore.java @@ -101,6 +101,13 @@ final class MigrationCandidateStore { } } + T withMaterial(ManagedMigrationConfigurationTransaction.CandidateRef reference, + MaterialReader reader) { + try (MigrationCandidateMaterial material = read(reference)) { + return reader.read(material); + } + } + ManagedMigrationConfigurationTransaction.DiscardOutcome discardExact( ManagedMigrationConfigurationTransaction.CandidateRef reference) throws IOException { CandidatePaths paths = paths(reference); @@ -263,4 +270,9 @@ final class MigrationCandidateStore { private enum EntryState { MISSING, PRESENT, UNSAFE } + @FunctionalInterface + interface MaterialReader { + T read(MigrationCandidateMaterial material); + } + } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/NioManagedFilePublisher.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/NioManagedFilePublisher.java index b56f641fe9..56892631f4 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/NioManagedFilePublisher.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/NioManagedFilePublisher.java @@ -25,6 +25,7 @@ import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.util.UUID; +import org.apache.hertzbeat.manager.setup.security.CommittedSetupFileDurabilityException; import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; /** Durable temp-write, file-fsync, replace, and directory-fsync publication. */ @@ -62,13 +63,26 @@ final class NioManagedFilePublisher implements ManagedFileIo.Publisher { @Override public void remove(Path target) throws IOException { if (Files.deleteIfExists(target)) { - operations.forceDirectory(target.toAbsolutePath().getParent()); + forceCommittedDirectory(target); } } + @Override + public void confirmDurability(Path target) throws IOException { + operations.forceDirectory(target.toAbsolutePath().getParent()); + } + private void replaceAndForce(Path source, Path target) throws IOException { operations.atomicReplace(source, target); - operations.forceDirectory(target.toAbsolutePath().getParent()); + forceCommittedDirectory(target); + } + + private void forceCommittedDirectory(Path target) throws IOException { + try { + confirmDurability(target); + } catch (IOException failure) { + throw new CommittedSetupFileDurabilityException(); + } } private static void writeAndForce(Path target, byte[] content) throws IOException { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/CommittedSetupFileDurabilityException.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/CommittedSetupFileDurabilityException.java index 2ec4ba1ae7..811b8a82b2 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/CommittedSetupFileDurabilityException.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/CommittedSetupFileDurabilityException.java @@ -9,7 +9,7 @@ package org.apache.hertzbeat.manager.setup.security; import java.io.IOException; -/** Rename committed the new file, but parent-directory durability could not be confirmed. */ +/** A filesystem entry mutation committed, but parent-directory durability could not be confirmed. */ public final class CommittedSetupFileDurabilityException extends IOException { public CommittedSetupFileDurabilityException() { diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/FileManagedConfigurationStoreTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/FileManagedConfigurationStoreTest.java index 63283bce62..2b7c7e51ea 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/FileManagedConfigurationStoreTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/FileManagedConfigurationStoreTest.java @@ -184,6 +184,11 @@ class FileManagedConfigurationStoreTest { public void remove(Path target) throws IOException { throw new IOException("injected removal failure"); } + + @Override + public void confirmDurability(Path target) throws IOException { + throw new IOException("injected durability failure"); + } }; FileManagedApplicationConfigStore failing = new FileManagedApplicationConfigStore(installationRoot, failingPublisher); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/FileManagedSnapshotStoreBufferTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/FileManagedSnapshotStoreBufferTest.java index 5e4c193e27..4dd1d8530f 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/FileManagedSnapshotStoreBufferTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/FileManagedSnapshotStoreBufferTest.java @@ -299,6 +299,11 @@ class FileManagedSnapshotStoreBufferTest { Files.deleteIfExists(target); } + @Override + public void confirmDurability(Path target) { + // The in-memory failure adapter has no delayed directory publication. + } + private List publishedTargets() { return publishedTargets; } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java index 61b19de622..03872057f2 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedConfigurationTransactionTest.java @@ -364,6 +364,11 @@ class ManagedConfigurationTransactionTest { public void remove(Path target) throws IOException { delegate.remove(target); } + + @Override + public void confirmDurability(Path target) throws IOException { + delegate.confirmDurability(target); + } } private static final class AtomicUnsupportedPublisher implements ManagedFileIo.Publisher { @@ -378,6 +383,11 @@ class ManagedConfigurationTransactionTest { public void remove(Path target) throws IOException { Files.deleteIfExists(target); } + + @Override + public void confirmDurability(Path target) { + // Atomic publication is rejected before durability confirmation is relevant. + } } /** Separate JVM entry point proving that the lock coordinates processes, not only instances. */ diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationActivationTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationActivationTest.java new file mode 100644 index 0000000000..2a71c6a837 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationActivationTest.java @@ -0,0 +1,509 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Stream; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class ManagedMigrationActivationTest { + + private static final String OPERATION = "migration-operation"; + private static final String CANDIDATE = "candidate-generation"; + private static final String IDENTITY = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + @TempDir + private Path installationRoot; + + @Test + void activatesAndRollsBackTheExactCandidateWithoutRemovingMigrationMaterial() throws Exception { + ManagedConfigurationTransaction setup = new ManagedConfigurationTransaction(installationRoot); + assertEquals(ManagedConfigurationTransaction.Outcome.APPLIED, setup.apply(bundle("base"))); + String baseGeneration = activeGeneration(installationRoot); + ManagedMigrationConfigurationTransaction migration = + new ManagedMigrationConfigurationTransaction(installationRoot); + ManagedMigrationConfigurationTransaction.CandidateRef reference = + migration.stage(OPERATION, CANDIDATE, baseGeneration, IDENTITY, bundle("next")); + + assertEquals(ManagedMigrationConfigurationTransaction.ActivationOutcome.ACTIVATED, + migration.activate(reference)); + assertEquals(CANDIDATE, activeGeneration(installationRoot)); + assertEquals(baseGeneration, new FileManagedApplicationConfigStore(installationRoot) + .readLastKnownGood().generation().orElseThrow()); + assertEquals(ManagedMigrationConfigurationTransaction.CandidateState.READY, + migration.inspect(reference).state()); + assertEquals(ManagedMigrationConfigurationTransaction.ActivationOutcome.ALREADY_ACTIVE, + migration.activate(reference)); + + assertEquals(ManagedMigrationConfigurationTransaction.RollbackOutcome.ROLLED_BACK, + migration.rollback(reference)); + assertEquals(baseGeneration, activeGeneration(installationRoot)); + assertEquals(ManagedMigrationConfigurationTransaction.CandidateState.READY, + migration.inspect(reference).state()); + assertEquals(ManagedMigrationConfigurationTransaction.RollbackOutcome.ALREADY_ROLLED_BACK, + migration.rollback(reference)); + } + + @Test + void laterActiveGenerationMakesActivationAndRollbackStaleWithoutWriting() throws Exception { + ManagedConfigurationTransaction setup = new ManagedConfigurationTransaction(installationRoot); + assertEquals(ManagedConfigurationTransaction.Outcome.APPLIED, setup.apply(bundle("base"))); + String baseGeneration = activeGeneration(installationRoot); + ManagedMigrationConfigurationTransaction migration = + new ManagedMigrationConfigurationTransaction(installationRoot); + ManagedMigrationConfigurationTransaction.CandidateRef reference = + migration.stage(OPERATION, CANDIDATE, baseGeneration, IDENTITY, bundle("next")); + assertEquals(ManagedMigrationConfigurationTransaction.ActivationOutcome.ACTIVATED, + migration.activate(reference)); + assertEquals(ManagedConfigurationTransaction.Outcome.APPLIED, setup.apply(bundle("later"))); + String laterGeneration = activeGeneration(installationRoot); + + assertEquals(ManagedMigrationConfigurationTransaction.ActivationOutcome.STALE, + migration.activate(reference)); + assertEquals(ManagedMigrationConfigurationTransaction.RollbackOutcome.STALE, + migration.rollback(reference)); + assertEquals(laterGeneration, activeGeneration(installationRoot)); + } + + @Test + void unrelatedSetupCandidateBlocksActivationAndIsNeverDeleted() throws Exception { + new ManagedConfigurationTransaction(installationRoot).apply(bundle("base")); + String baseGeneration = activeGeneration(installationRoot); + ManagedMigrationConfigurationTransaction migration = + new ManagedMigrationConfigurationTransaction(installationRoot); + ManagedMigrationConfigurationTransaction.CandidateRef reference = + migration.stage(OPERATION, CANDIDATE, baseGeneration, IDENTITY, bundle("next")); + FileManagedApplicationConfigStore applications = new FileManagedApplicationConfigStore(installationRoot); + applications.stageCandidate(configuration("foreign"), "foreign-generation"); + + assertEquals(ManagedMigrationConfigurationTransaction.ActivationOutcome.RECOVERY_REQUIRED, + migration.activate(reference)); + assertEquals("foreign-generation", applications.readCandidate().generation().orElseThrow()); + assertEquals(baseGeneration, applications.readActive().generation().orElseThrow()); + } + + @ParameterizedTest + @MethodSource("activationPublicationFailures") + void reconcilesActivationWhenPublicationCommitsBeforeReportingFailure( + String fileName, Operation operation) throws Exception { + Path root = installationRoot.resolve(fileName.replace('.', '-')); + new ManagedConfigurationTransaction(root).apply(bundle("base")); + String baseGeneration = activeGeneration(root); + CommitThenFailPublisher publisher = new CommitThenFailPublisher(fileName, operation); + ManagedMigrationActivation activation = new ManagedMigrationActivation( + new FileManagedApplicationConfigStore(root, publisher), + new FileManagedSecretStore(root, publisher)); + + try (MigrationCandidateMaterial material = material(baseGeneration)) { + assertEquals(ManagedMigrationConfigurationTransaction.ActivationOutcome.RECOVERY_REQUIRED, + activation.activate(material)); + ManagedMigrationActivation retry = new ManagedMigrationActivation( + new FileManagedApplicationConfigStore(root), new FileManagedSecretStore(root)); + assertTrue(retry.activate(material) + != ManagedMigrationConfigurationTransaction.ActivationOutcome.RECOVERY_REQUIRED); + } + + assertTrue(publisher.failed()); + assertEquals(CANDIDATE, activeGeneration(root)); + assertEquals(CANDIDATE, new FileManagedSecretStore(root).readActive().generation().orElseThrow()); + assertEquals(baseGeneration, new FileManagedApplicationConfigStore(root) + .readLastKnownGood().generation().orElseThrow()); + } + + @ParameterizedTest + @MethodSource("rollbackPublicationFailures") + void reconcilesRollbackWhenActiveReplaceCommitsBeforeReportingFailure(String fileName) throws Exception { + Path root = installationRoot.resolve(fileName.replace('.', '-')); + new ManagedConfigurationTransaction(root).apply(bundle("base")); + String baseGeneration = activeGeneration(root); + try (MigrationCandidateMaterial material = material(baseGeneration)) { + ManagedMigrationActivation initial = new ManagedMigrationActivation( + new FileManagedApplicationConfigStore(root), new FileManagedSecretStore(root)); + assertEquals(ManagedMigrationConfigurationTransaction.ActivationOutcome.ACTIVATED, + initial.activate(material)); + CommitThenFailPublisher publisher = new CommitThenFailPublisher(fileName, Operation.PUBLISH); + ManagedMigrationActivation rollback = new ManagedMigrationActivation( + new FileManagedApplicationConfigStore(root, publisher), + new FileManagedSecretStore(root, publisher)); + + assertEquals(ManagedMigrationConfigurationTransaction.RollbackOutcome.RECOVERY_REQUIRED, + rollback.rollback(material)); + assertTrue(publisher.failed()); + ManagedMigrationActivation retry = new ManagedMigrationActivation( + new FileManagedApplicationConfigStore(root), new FileManagedSecretStore(root)); + assertTrue(retry.rollback(material) + != ManagedMigrationConfigurationTransaction.RollbackOutcome.RECOVERY_REQUIRED); + } + + assertEquals(baseGeneration, activeGeneration(root)); + assertEquals(baseGeneration, new FileManagedSecretStore(root).readActive().generation().orElseThrow()); + } + + @Test + void preCommitFailureStopsActivationUntilAnExplicitRetry() throws Exception { + new ManagedConfigurationTransaction(installationRoot).apply(bundle("base")); + String baseGeneration = activeGeneration(installationRoot); + CommitThenFailPublisher publisher = new CommitThenFailPublisher( + "managed-application.yml.candidate", Operation.PUBLISH, false); + ManagedMigrationActivation activation = new ManagedMigrationActivation( + new FileManagedApplicationConfigStore(installationRoot, publisher), + new FileManagedSecretStore(installationRoot, publisher)); + + try (MigrationCandidateMaterial material = material(baseGeneration)) { + assertEquals(ManagedMigrationConfigurationTransaction.ActivationOutcome.RECOVERY_REQUIRED, + activation.activate(material)); + assertEquals(CandidateState.MISSING, + new FileManagedApplicationConfigStore(installationRoot).readCandidate().state()); + assertEquals(baseGeneration, activeGeneration(installationRoot)); + ManagedMigrationActivation retry = new ManagedMigrationActivation( + new FileManagedApplicationConfigStore(installationRoot), + new FileManagedSecretStore(installationRoot)); + assertEquals(ManagedMigrationConfigurationTransaction.ActivationOutcome.ACTIVATED, + retry.activate(material)); + } + } + + @Test + void setupRecoveryRollsBackAnInterruptedExactStageToTheCompleteBasePair() throws Exception { + new ManagedConfigurationTransaction(installationRoot).apply(bundle("base")); + String baseGeneration = activeGeneration(installationRoot); + CommitThenFailPublisher publisher = new CommitThenFailPublisher( + "managed-application.yml.candidate", Operation.PUBLISH); + ManagedMigrationActivation activation = new ManagedMigrationActivation( + new FileManagedApplicationConfigStore(installationRoot, publisher), + new FileManagedSecretStore(installationRoot, publisher)); + try (MigrationCandidateMaterial material = material(baseGeneration)) { + assertEquals(ManagedMigrationConfigurationTransaction.ActivationOutcome.RECOVERY_REQUIRED, + activation.activate(material)); + } + + assertEquals(ManagedConfigurationTransaction.Outcome.ROLLED_BACK, + new ManagedConfigurationTransaction(installationRoot).recover()); + assertEquals(baseGeneration, activeGeneration(installationRoot)); + assertEquals(baseGeneration, new FileManagedSecretStore(installationRoot) + .readActive().generation().orElseThrow()); + } + + @Test + void setupRecoveryCompletesAnInterruptedExactPromotionAsOneTargetPair() throws Exception { + new ManagedConfigurationTransaction(installationRoot).apply(bundle("base")); + String baseGeneration = activeGeneration(installationRoot); + CommitThenFailPublisher publisher = new CommitThenFailPublisher( + "managed-application.yml", Operation.PUBLISH); + ManagedMigrationActivation activation = new ManagedMigrationActivation( + new FileManagedApplicationConfigStore(installationRoot, publisher), + new FileManagedSecretStore(installationRoot, publisher)); + try (MigrationCandidateMaterial material = material(baseGeneration)) { + assertEquals(ManagedMigrationConfigurationTransaction.ActivationOutcome.RECOVERY_REQUIRED, + activation.activate(material)); + } + + assertEquals(ManagedConfigurationTransaction.Outcome.APPLIED, + new ManagedConfigurationTransaction(installationRoot).recover()); + assertEquals(CANDIDATE, activeGeneration(installationRoot)); + assertEquals(CANDIDATE, new FileManagedSecretStore(installationRoot) + .readActive().generation().orElseThrow()); + } + + @Test + void setupRecoveryCompletesAnInterruptedExactRollbackAsOneBasePair() throws Exception { + new ManagedConfigurationTransaction(installationRoot).apply(bundle("base")); + String baseGeneration = activeGeneration(installationRoot); + try (MigrationCandidateMaterial material = material(baseGeneration)) { + ManagedMigrationActivation activation = new ManagedMigrationActivation( + new FileManagedApplicationConfigStore(installationRoot), + new FileManagedSecretStore(installationRoot)); + assertEquals(ManagedMigrationConfigurationTransaction.ActivationOutcome.ACTIVATED, + activation.activate(material)); + CommitThenFailPublisher publisher = new CommitThenFailPublisher( + "managed-secrets.properties", Operation.PUBLISH); + ManagedMigrationActivation rollback = new ManagedMigrationActivation( + new FileManagedApplicationConfigStore(installationRoot, publisher), + new FileManagedSecretStore(installationRoot, publisher)); + assertEquals(ManagedMigrationConfigurationTransaction.RollbackOutcome.RECOVERY_REQUIRED, + rollback.rollback(material)); + } + + assertEquals(ManagedConfigurationTransaction.Outcome.ROLLED_BACK, + new ManagedConfigurationTransaction(installationRoot).recover()); + assertEquals(baseGeneration, activeGeneration(installationRoot)); + assertEquals(baseGeneration, new FileManagedSecretStore(installationRoot) + .readActive().generation().orElseThrow()); + } + + @Test + void retryConfirmsDurabilityAfterFinalRollbackRenameReportedDirectoryForceFailure() throws Exception { + new ManagedConfigurationTransaction(installationRoot).apply(bundle("base")); + String baseGeneration = activeGeneration(installationRoot); + try (MigrationCandidateMaterial material = material(baseGeneration)) { + ManagedMigrationActivation activation = new ManagedMigrationActivation( + new FileManagedApplicationConfigStore(installationRoot), + new FileManagedSecretStore(installationRoot)); + assertEquals(ManagedMigrationConfigurationTransaction.ActivationOutcome.ACTIVATED, + activation.activate(material)); + FailRollbackAndFirstConfirmationOperations operations = + new FailRollbackAndFirstConfirmationOperations(); + NioManagedFilePublisher publisher = new NioManagedFilePublisher(operations); + ManagedMigrationActivation rollback = new ManagedMigrationActivation( + new FileManagedApplicationConfigStore(installationRoot, publisher), + new FileManagedSecretStore(installationRoot, publisher)); + + assertEquals(ManagedMigrationConfigurationTransaction.RollbackOutcome.RECOVERY_REQUIRED, + rollback.rollback(material)); + assertEquals(baseGeneration, activeGeneration(installationRoot)); + assertEquals(baseGeneration, new FileManagedSecretStore(installationRoot) + .readActive().generation().orElseThrow()); + assertEquals(ManagedMigrationConfigurationTransaction.RollbackOutcome.RECOVERY_REQUIRED, + rollback.rollback(material)); + assertEquals(ManagedMigrationConfigurationTransaction.RollbackOutcome.ALREADY_ROLLED_BACK, + rollback.rollback(material)); + assertEquals(5, operations.forces()); + } + } + + @Test + void corruptFixedCandidateBlocksActivationWithoutOverwriteOrDeletion() throws Exception { + new ManagedConfigurationTransaction(installationRoot).apply(bundle("base")); + String baseGeneration = activeGeneration(installationRoot); + Path fixedCandidate = installationRoot.resolve("data/config/managed-application.yml.candidate"); + Files.writeString(fixedCandidate, "not-a-managed-document"); + byte[] original = Files.readAllBytes(fixedCandidate); + ManagedMigrationActivation activation = new ManagedMigrationActivation( + new FileManagedApplicationConfigStore(installationRoot), + new FileManagedSecretStore(installationRoot)); + + try (MigrationCandidateMaterial material = material(baseGeneration)) { + assertEquals(ManagedMigrationConfigurationTransaction.ActivationOutcome.RECOVERY_REQUIRED, + activation.activate(material)); + } + + assertArrayEquals(original, Files.readAllBytes(fixedCandidate)); + assertEquals(baseGeneration, activeGeneration(installationRoot)); + } + + @ParameterizedTest + @MethodSource("invalidRollbackLastKnownGood") + void rollbackRejectsIncompleteOrCorruptLastKnownGood(String fileName, boolean remove) throws Exception { + Path root = installationRoot.resolve(fileName.replace('.', '-') + remove); + new ManagedConfigurationTransaction(root).apply(bundle("base")); + String baseGeneration = activeGeneration(root); + ManagedMigrationActivation activation = new ManagedMigrationActivation( + new FileManagedApplicationConfigStore(root), new FileManagedSecretStore(root)); + try (MigrationCandidateMaterial material = material(baseGeneration)) { + assertEquals(ManagedMigrationConfigurationTransaction.ActivationOutcome.ACTIVATED, + activation.activate(material)); + Path lkg = root.resolve("data/config/" + fileName); + if (remove) { + Files.delete(lkg); + } else { + Files.writeString(lkg, "corrupt"); + } + assertEquals(ManagedMigrationConfigurationTransaction.RollbackOutcome.RECOVERY_REQUIRED, + activation.rollback(material)); + } + assertEquals(CANDIDATE, activeGeneration(root)); + } + + @Test + void rollbackRejectsAnAggregateInvalidLastKnownGoodPair() throws Exception { + new ManagedConfigurationTransaction(installationRoot).apply(bundle("base")); + String baseGeneration = activeGeneration(installationRoot); + ManagedMigrationActivation activation = new ManagedMigrationActivation( + new FileManagedApplicationConfigStore(installationRoot), + new FileManagedSecretStore(installationRoot)); + try (MigrationCandidateMaterial material = material(baseGeneration)) { + assertEquals(ManagedMigrationConfigurationTransaction.ActivationOutcome.ACTIVATED, + activation.activate(material)); + try (ManagedSecrets incomplete = new ManagedSecrets( + SecretValue.of("database-base"), Optional.empty(), Optional.empty())) { + byte[] encoded = new SecretConfigDocumentCodec().encode(incomplete, baseGeneration); + try { + new NioManagedFilePublisher().publish(installationRoot.resolve( + "data/config/managed-secrets.properties.last-known-good"), encoded, true); + } finally { + Arrays.fill(encoded, (byte) 0); + } + } + + assertEquals(ManagedMigrationConfigurationTransaction.RollbackOutcome.RECOVERY_REQUIRED, + activation.rollback(material)); + } + assertEquals(CANDIDATE, activeGeneration(installationRoot)); + } + + @ParameterizedTest + @MethodSource("rejectedExactOutcomes") + void exactStepNeverIgnoresStaleOrRecoveryOutcome(ExactSnapshotOutcome outcome) { + assertThrows(IOException.class, () -> MigrationActivationStepExecutor.requireApplied(outcome)); + } + + private static Stream activationPublicationFailures() { + return Stream.of( + Arguments.of("managed-application.yml.candidate", Operation.PUBLISH), + Arguments.of("managed-secrets.properties.candidate", Operation.PUBLISH), + Arguments.of("managed-application.yml.last-known-good", Operation.PUBLISH), + Arguments.of("managed-application.yml", Operation.PUBLISH), + Arguments.of("managed-secrets.properties.last-known-good", Operation.PUBLISH), + Arguments.of("managed-secrets.properties", Operation.PUBLISH), + Arguments.of("managed-application.yml.candidate", Operation.REMOVE), + Arguments.of("managed-secrets.properties.candidate", Operation.REMOVE)); + } + + private static Stream rollbackPublicationFailures() { + return Stream.of("managed-secrets.properties", "managed-application.yml"); + } + + private static Stream invalidRollbackLastKnownGood() { + return Stream.of( + Arguments.of("managed-application.yml.last-known-good", true), + Arguments.of("managed-secrets.properties.last-known-good", true), + Arguments.of("managed-application.yml.last-known-good", false), + Arguments.of("managed-secrets.properties.last-known-good", false)); + } + + private static Stream rejectedExactOutcomes() { + return Stream.of(ExactSnapshotOutcome.STALE, ExactSnapshotOutcome.RECOVERY_REQUIRED); + } + + private static MigrationCandidateMaterial material(String baseGeneration) { + MigrationCandidateManifest manifest = new MigrationCandidateManifest( + OPERATION, CANDIDATE, baseGeneration, IDENTITY); + ManagedConfigurationBundle bundle = bundle("next"); + return MigrationCandidateMaterial.ready(manifest, bundle.application(), bundle.secrets()); + } + + private static String activeGeneration(Path root) { + return new FileManagedApplicationConfigStore(root).readActive().generation().orElseThrow(); + } + + private static ManagedConfigurationBundle bundle(String suffix) { + return new ManagedConfigurationBundle(configuration(suffix), + new ManagedSecrets(SecretValue.of("database-" + suffix), + Optional.of(SecretValue.of("telemetry-" + suffix)), + Optional.of(SecretValue.of("mail-" + suffix)))); + } + + private static ManagedApplicationConfig configuration(String suffix) { + ManagedOptionalConfiguration.MailSettings mail = new ManagedOptionalConfiguration.MailSettings( + "smtp.example", 587, SetupApiContract.MailSecurity.STARTTLS, + Optional.of("mailer"), "alerts@example.org"); + return new ManagedApplicationConfig( + new MetadataDatabaseSettings(SetupApiContract.MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://db/" + suffix, "hertzbeat"), + new GreptimeSettings(new GreptimeEndpoints("greptime:4001", "http://greptime:4000"), + "public", Optional.of("telemetry-user")), + new ManagedOptionalConfiguration(Optional.empty(), Optional.empty(), Optional.of(mail))); + } + + private enum Operation { PUBLISH, REMOVE } + + private static final class CommitThenFailPublisher implements ManagedFileIo.Publisher { + private final ManagedFileIo.Publisher delegate = new NioManagedFilePublisher(); + private final String fileName; + private final Operation operation; + private final AtomicBoolean failed = new AtomicBoolean(); + + private final boolean commitBeforeFailure; + + private CommitThenFailPublisher(String fileName, Operation operation) { + this(fileName, operation, true); + } + + private CommitThenFailPublisher(String fileName, Operation operation, boolean commitBeforeFailure) { + this.fileName = fileName; + this.operation = operation; + this.commitBeforeFailure = commitBeforeFailure; + } + + @Override + public void publish(Path target, byte[] content, boolean ownerOnly) throws IOException { + failBeforeCommit(target, Operation.PUBLISH); + delegate.publish(target, content, ownerOnly); + failOnce(target, Operation.PUBLISH); + } + + @Override + public void remove(Path target) throws IOException { + failBeforeCommit(target, Operation.REMOVE); + delegate.remove(target); + failOnce(target, Operation.REMOVE); + } + + @Override + public void confirmDurability(Path target) throws IOException { + delegate.confirmDurability(target); + } + + private void failBeforeCommit(Path target, Operation actual) throws IOException { + if (!commitBeforeFailure) { + fail(target, actual); + } + } + + private void failOnce(Path target, Operation actual) throws IOException { + if (commitBeforeFailure) { + fail(target, actual); + } + } + + private void fail(Path target, Operation actual) throws IOException { + if (operation == actual && target.getFileName().toString().equals(fileName) + && failed.compareAndSet(false, true)) { + throw new IOException("simulated publication failure"); + } + } + + private boolean failed() { + return failed.get(); + } + } + + private static final class FailRollbackAndFirstConfirmationOperations implements ManagedFileIo.Operations { + private int forces; + + @Override + public void atomicReplace(Path source, Path target) throws IOException { + Files.move(source, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } + + @Override + public void forceDirectory(Path directory) throws IOException { + forces++; + if (forces == 2 || forces == 3) { + throw new IOException("simulated directory force failure"); + } + if (Files.getFileStore(directory).supportsFileAttributeView("posix")) { + try (FileChannel channel = FileChannel.open(directory, StandardOpenOption.READ)) { + channel.force(true); + } + } + } + + private int forces() { + return forces; + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/MigrationActivationClassifierTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/MigrationActivationClassifierTest.java new file mode 100644 index 0000000000..949e7cac74 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/MigrationActivationClassifierTest.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract; +import org.junit.jupiter.api.Test; + +class MigrationActivationClassifierTest { + + private final MigrationActivationClassifier classifier = new MigrationActivationClassifier(); + + @Test + void splitLaterAndBaseActivePairRequiresRecovery() { + try (MigrationActivationSnapshots snapshots = snapshots( + CandidateRead.valid(configuration("later"), "later-generation"), + CandidateRead.valid(secrets("base"), "base-generation"))) { + assertRecovery(snapshots); + } + } + + @Test + void twoDifferentLaterGenerationsRequireRecovery() { + try (MigrationActivationSnapshots snapshots = snapshots( + CandidateRead.valid(configuration("later"), "later-application"), + CandidateRead.valid(secrets("later"), "later-secrets"))) { + assertRecovery(snapshots); + } + } + + @Test + void aggregateInvalidLaterPairRequiresRecovery() { + ManagedSecrets incomplete = new ManagedSecrets( + SecretValue.of("database-later"), Optional.empty(), Optional.empty()); + try (MigrationActivationSnapshots snapshots = snapshots( + CandidateRead.valid(configuration("later"), "later-generation"), + CandidateRead.valid(incomplete, "later-generation"))) { + assertRecovery(snapshots); + } + } + + @Test + void completeAggregateValidLaterPairIsStale() { + try (MigrationActivationSnapshots snapshots = snapshots( + CandidateRead.valid(configuration("later"), "later-generation"), + CandidateRead.valid(secrets("later"), "later-generation")); + ManagedSecrets targetSecrets = secrets("target")) { + MigrationActivationCandidate candidate = new MigrationActivationCandidate( + "candidate-generation", "base-generation", configuration("target"), targetSecrets); + assertEquals(MigrationActivationClassifier.Decision.STALE, + classifier.activation(candidate, snapshots)); + assertEquals(MigrationActivationClassifier.Decision.STALE, + classifier.rollback(candidate, snapshots)); + } + } + + @Test + void rollbackRequiresTheBaseSideToMatchItsLastKnownGoodSnapshot() { + try (ManagedSecrets targetSecrets = secrets("target"); + MigrationActivationSnapshots snapshots = new MigrationActivationSnapshots( + CandidateRead.valid(configuration("target"), "candidate-generation"), + CandidateRead.missing(), CandidateRead.valid(configuration("base"), "base-generation"), + CandidateRead.valid(secrets("different"), "base-generation"), + CandidateRead.missing(), CandidateRead.valid(secrets("base"), "base-generation"))) { + MigrationActivationCandidate candidate = new MigrationActivationCandidate( + "candidate-generation", "base-generation", configuration("target"), targetSecrets); + assertEquals(MigrationActivationClassifier.Decision.RECOVERY_REQUIRED, + classifier.rollback(candidate, snapshots)); + } + } + + private void assertRecovery(MigrationActivationSnapshots snapshots) { + try (ManagedSecrets targetSecrets = secrets("target")) { + MigrationActivationCandidate candidate = new MigrationActivationCandidate( + "candidate-generation", "base-generation", configuration("target"), targetSecrets); + assertEquals(MigrationActivationClassifier.Decision.RECOVERY_REQUIRED, + classifier.activation(candidate, snapshots)); + assertEquals(MigrationActivationClassifier.Decision.RECOVERY_REQUIRED, + classifier.rollback(candidate, snapshots)); + } + } + + private static MigrationActivationSnapshots snapshots( + CandidateRead activeApplication, + CandidateRead activeSecrets) { + return new MigrationActivationSnapshots(activeApplication, CandidateRead.missing(), CandidateRead.missing(), + activeSecrets, CandidateRead.missing(), CandidateRead.missing()); + } + + private static ManagedSecrets secrets(String suffix) { + return new ManagedSecrets(SecretValue.of("database-" + suffix), + Optional.of(SecretValue.of("telemetry-" + suffix)), + Optional.of(SecretValue.of("mail-" + suffix))); + } + + private static ManagedApplicationConfig configuration(String suffix) { + ManagedOptionalConfiguration.MailSettings mail = new ManagedOptionalConfiguration.MailSettings( + "smtp.example", 587, SetupApiContract.MailSecurity.STARTTLS, + Optional.of("mailer"), "alerts@example.org"); + return new ManagedApplicationConfig( + new MetadataDatabaseSettings(SetupApiContract.MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://db/" + suffix, "hertzbeat"), + new GreptimeSettings(new GreptimeEndpoints("greptime:4001", "http://greptime:4000"), + "public", Optional.of("telemetry-user")), + new ManagedOptionalConfiguration(Optional.empty(), Optional.empty(), Optional.of(mail))); + } +} From f5b4d7401c3f81f67d4fee395be517a2230cc02b Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 08:20:27 +0800 Subject: [PATCH 44/71] Bound metadata migration execution --- .../JdbcMetadataMigrationAttempt.java | 258 +++++++++++ .../JdbcMetadataMigrationDeadline.java | 70 +++ .../JdbcMetadataMigrationExecutor.java | 218 +++++++++ .../JdbcMigrationConnectionScope.java | 183 ++++++++ ...adataMigrationConnectionLifecycleTest.java | 287 ++++++++++++ .../JdbcMetadataMigrationDeadlineTest.java | 53 +++ ...etadataMigrationExecutorLifecycleTest.java | 380 ++++++++++++++++ .../JdbcMetadataMigrationExecutorTest.java | 421 ++++++++++++++++++ 8 files changed, 1870 insertions(+) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationAttempt.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationDeadline.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationExecutor.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMigrationConnectionScope.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationConnectionLifecycleTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationDeadlineTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationExecutorLifecycleTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationExecutorTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationAttempt.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationAttempt.java new file mode 100644 index 0000000000..152c8f7146 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationAttempt.java @@ -0,0 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Owns one worker's connection-deadline mutation, result, cancellation, and exit proof. */ +final class JdbcMetadataMigrationAttempt { + + private final Object stateLock = new Object(); + private final CountDownLatch workerExited = new CountDownLatch(1); + private final Connection source; + private final Connection target; + private final MetadataDatabaseKind targetKind; + private final JdbcMetadataMigrationDeadline deadline; + private final JdbcMigrationConnectionScope connections; + private final JdbcMetadataMigrationExecutor.MigrationWork work; + private final MetadataMigrationProgressSink progress; + private volatile State state = State.RUNNING; + private boolean workerStarted; + private Thread workerThread; + private Throwable resultFailure; + + JdbcMetadataMigrationAttempt( + Connection source, + Connection target, + MetadataDatabaseKind targetKind, + JdbcMetadataMigrationDeadline deadline, + JdbcMigrationConnectionScope connections, + JdbcMetadataMigrationExecutor.MigrationWork work, + MetadataMigrationProgressSink progress) { + this.source = source; + this.target = target; + this.targetKind = targetKind; + this.deadline = deadline; + this.connections = connections; + this.work = work; + this.progress = progress; + } + + void run() { + if (!beginWorker()) { + workerExited.countDown(); + return; + } + Throwable completion = null; + boolean invalidate = false; + try { + connections.configure(); + completion = runMigration(); + if (!abandoned() && !connections.restore()) { + invalidate = true; + } + } catch (SQLException | RuntimeException configurationFailure) { + completion = new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT); + invalidate = true; + } catch (Error lifecycleFailure) { + completion = lifecycleFailure; + invalidate = true; + } finally { + if (invalidate) { + completion = invalidateConnections(completion); + } + try { + complete(completion); + } catch (Error lifecycleFailure) { + forceFatalCompletion(lifecycleFailure); + } finally { + workerExited.countDown(); + } + } + } + + private Throwable runMigration() { + try { + work.migrate(source, target, targetKind, deadline.remainingDuration(), this::reportProgress); + return null; + } catch (MetadataMigrationException stable) { + return stable; + } catch (Error fatal) { + return fatal; + } catch (RuntimeException unexpected) { + return new MetadataMigrationException(MetadataMigrationErrorCode.COPY); + } + } + + private Throwable invalidateConnections(Throwable completion) { + Throwable result = completion; + try { + abortConnections(); + } catch (Error lifecycleFailure) { + result = lifecycleFailure; + } + try { + closeInvalidatedConnections(); + } catch (Error lifecycleFailure) { + result = lifecycleFailure; + } + return result; + } + + boolean awaitExit(long timeoutNanos) throws InterruptedException { + return workerExited.await(timeoutNanos, TimeUnit.NANOSECONDS); + } + + boolean awaitExitUninterruptibly() { + boolean interrupted = false; + while (workerExited.getCount() > 0) { + try { + workerExited.await(); + } catch (InterruptedException ignored) { + interrupted = true; + } + } + return interrupted; + } + + Abandonment abandon() { + synchronized (stateLock) { + if (state != State.RUNNING) { + return new Abandonment(false, workerStarted); + } + state = State.ABANDONED; + return new Abandonment(true, workerStarted); + } + } + + void completeNeverStartedWorker() { + workerExited.countDown(); + } + + void abortConnections() { + Error failure = connections.abort(); + if (failure != null) { + forceFatalCompletion(failure); + } + } + + void closeInvalidatedConnections() { + Error failure = connections.closeInvalidated(); + if (failure != null) { + forceFatalCompletion(failure); + throw failure; + } + } + + void rethrowFailure() { + Throwable failure; + synchronized (stateLock) { + failure = resultFailure; + } + if (failure instanceof Error error) { + throw error; + } + if (failure instanceof MetadataMigrationException stable) { + throw stable; + } + if (failure != null) { + throw new MetadataMigrationException(MetadataMigrationErrorCode.COPY); + } + } + + void rethrowFatalOrOutcomeUnknown() { + Throwable failure; + synchronized (stateLock) { + failure = resultFailure; + } + if (failure instanceof Error fatal) { + throw fatal; + } + if (failure instanceof MetadataMigrationException stable + && (stable.code() == MetadataMigrationErrorCode.COMMIT_OUTCOME_UNKNOWN + || stable.code() == MetadataMigrationErrorCode.ROLLBACK_OUTCOME_UNKNOWN)) { + throw stable; + } + } + + private boolean beginWorker() { + synchronized (stateLock) { + workerStarted = true; + workerThread = Thread.currentThread(); + return state != State.ABANDONED; + } + } + + boolean isCurrentWorkerThread() { + synchronized (stateLock) { + return workerThread == Thread.currentThread(); + } + } + + private void complete(Throwable failure) { + synchronized (stateLock) { + if (state == State.RUNNING) { + if (outranks(failure, resultFailure)) { + resultFailure = failure; + } + state = State.RESULT_READY; + } else if (state == State.ABANDONED && outranks(failure, resultFailure)) { + resultFailure = failure; + } + } + } + + private void forceFatalCompletion(Error failure) { + synchronized (stateLock) { + if (outranks(failure, resultFailure)) { + resultFailure = failure; + } + if (state == State.RUNNING) { + state = State.RESULT_READY; + } + } + } + + private void reportProgress(MetadataMigrationStage stage, int percent) { + if (state == State.RUNNING) { + progress.report(stage, percent); + } + } + + boolean abandoned() { + synchronized (stateLock) { + return state == State.ABANDONED; + } + } + + private static boolean outranks(Throwable candidate, Throwable current) { + return failurePriority(candidate) > failurePriority(current); + } + + private static int failurePriority(Throwable failure) { + if (failure instanceof Error) { + return 2; + } + if (failure instanceof MetadataMigrationException stable) { + if (stable.code() == MetadataMigrationErrorCode.COMMIT_OUTCOME_UNKNOWN + || stable.code() == MetadataMigrationErrorCode.ROLLBACK_OUTCOME_UNKNOWN) { + return 1; + } + } + return failure == null ? -1 : 0; + } + + record Abandonment(boolean abandoned, boolean workerStarted) { + } + + private enum State { RUNNING, RESULT_READY, ABANDONED } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationDeadline.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationDeadline.java new file mode 100644 index 0000000000..3ec0d554a7 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationDeadline.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.time.Duration; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.function.LongSupplier; + +/** One monotonic budget shared by scheduling, network configuration, and copy execution. */ +final class JdbcMetadataMigrationDeadline { + + private final long startedAtNanos; + private final long timeoutNanos; + private final LongSupplier ticker; + + private JdbcMetadataMigrationDeadline(long startedAtNanos, long timeoutNanos, LongSupplier ticker) { + this.startedAtNanos = startedAtNanos; + this.timeoutNanos = timeoutNanos; + this.ticker = ticker; + } + + static JdbcMetadataMigrationDeadline start(Duration timeout, LongSupplier ticker) { + Objects.requireNonNull(timeout, "timeout"); + Objects.requireNonNull(ticker, "ticker"); + if (timeout.isZero() || timeout.isNegative()) { + throw new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT); + } + long timeoutNanos; + try { + timeoutNanos = timeout.toNanos(); + } catch (ArithmeticException overflow) { + timeoutNanos = Long.MAX_VALUE; + } + return new JdbcMetadataMigrationDeadline(ticker.getAsLong(), timeoutNanos, ticker); + } + + long remainingNanos() { + long elapsed = ticker.getAsLong() - startedAtNanos; + if (elapsed <= 0) { + return timeoutNanos; + } + return elapsed >= timeoutNanos ? 0 : timeoutNanos - elapsed; + } + + Duration remainingDuration() { + long remaining = remainingNanos(); + if (remaining <= 0) { + throw new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT); + } + return Duration.ofNanos(remaining); + } + + int remainingMillis() { + long remaining = remainingNanos(); + if (remaining <= 0) { + throw new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT); + } + long millis = TimeUnit.NANOSECONDS.toMillis(remaining); + if (remaining % TimeUnit.MILLISECONDS.toNanos(1) != 0) { + millis++; + } + return (int) Math.min(Integer.MAX_VALUE, Math.max(1, millis)); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationExecutor.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationExecutor.java new file mode 100644 index 0000000000..ae4f11865a --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationExecutor.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.time.Duration; +import java.util.Objects; +import java.util.concurrent.Executor; +import java.util.concurrent.Future; +import java.util.concurrent.FutureTask; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.function.LongSupplier; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** + * Runs one caller-owned JDBC metadata copy with socket deadlines and an outer abort watchdog. + * + *

A deadline requests cancellation and aborts both connections, but its hard return bound still + * depends on the JDBC driver honoring {@link Connection#abort(Executor)}. This method does not + * return until the migration worker, including rollback and connection-state restoration, has + * actually exited. A deadline-invalidated connection is closed and must not be reused. + */ +public final class JdbcMetadataMigrationExecutor implements AutoCloseable { + + private final Object lifecycleLock = new Object(); + private final ThreadPoolExecutor worker; + private final ThreadPoolExecutor abortWorker; + private final Executor networkExecutor; + private final MigrationWork work; + private final LongSupplier ticker; + private ActiveExecution active; + private boolean closed; + + public JdbcMetadataMigrationExecutor() { + this(worker(), abortWorker(), Runnable::run, new JdbcMetadataMigration()::migrate, System::nanoTime); + } + + JdbcMetadataMigrationExecutor( + ThreadPoolExecutor worker, Executor networkExecutor, MigrationWork work) { + this(worker, abortWorker(), networkExecutor, work, System::nanoTime); + } + + JdbcMetadataMigrationExecutor( + ThreadPoolExecutor worker, + ThreadPoolExecutor abortWorker, + Executor networkExecutor, + MigrationWork work, + LongSupplier ticker) { + this.worker = Objects.requireNonNull(worker, "worker"); + this.abortWorker = Objects.requireNonNull(abortWorker, "abortWorker"); + this.networkExecutor = Objects.requireNonNull(networkExecutor, "networkExecutor"); + this.work = Objects.requireNonNull(work, "work"); + this.ticker = Objects.requireNonNull(ticker, "ticker"); + } + + /** Executes without accepting or retaining any JDBC URL, username, or credential. */ + public void execute( + Connection source, + Connection target, + MetadataDatabaseKind targetKind, + Duration timeout, + MetadataMigrationProgressSink progress) { + Objects.requireNonNull(source, "source"); + Objects.requireNonNull(target, "target"); + Objects.requireNonNull(targetKind, "targetKind"); + Objects.requireNonNull(progress, "progress"); + JdbcMetadataMigrationDeadline deadline = JdbcMetadataMigrationDeadline.start(timeout, ticker); + JdbcMigrationConnectionScope connections = new JdbcMigrationConnectionScope( + source, target, deadline, networkExecutor, abortWorker); + JdbcMetadataMigrationAttempt attempt = new JdbcMetadataMigrationAttempt( + source, target, targetKind, deadline, connections, work, progress); + ActiveExecution execution; + synchronized (lifecycleLock) { + if (closed || active != null) { + throw timeoutFailure(); + } + FutureTask submitted = new FutureTask<>(attempt::run, null); + execution = new ActiveExecution(attempt, submitted); + active = execution; + try { + worker.execute(submitted); + } catch (RuntimeException unexpected) { + active = null; + submitted.cancel(false); + throw timeoutFailure(); + } catch (Error fatal) { + active = null; + submitted.cancel(false); + throw fatal; + } + } + + boolean callerInterrupted = false; + try { + boolean abandoned = false; + try { + long remaining = deadline.remainingNanos(); + if (remaining <= 0 || !attempt.awaitExit(remaining)) { + abandoned = stop(execution); + } + } catch (InterruptedException interrupted) { + callerInterrupted = true; + abandoned = stop(execution); + } + callerInterrupted |= attempt.awaitExitUninterruptibly(); + abandoned |= attempt.abandoned(); + if (abandoned) { + attempt.closeInvalidatedConnections(); + } + if (abandoned) { + attempt.rethrowFatalOrOutcomeUnknown(); + throw timeoutFailure(); + } + attempt.rethrowFailure(); + } finally { + if (callerInterrupted) { + Thread.currentThread().interrupt(); + } + synchronized (lifecycleLock) { + if (active == execution) { + active = null; + } + } + } + } + + /** + * Stops an active copy and joins its abort and connection disposal. + * + * @throws IllegalStateException when invoked by the active copy worker or its progress callback + */ + @Override + public void close() { + ActiveExecution execution; + synchronized (lifecycleLock) { + execution = active; + if (execution != null && execution.attempt().isCurrentWorkerThread()) { + throw new IllegalStateException("Migration executor cannot be closed by its copy worker"); + } + closed = true; + } + boolean interrupted = false; + try { + if (execution != null) { + stop(execution); + interrupted = execution.attempt().awaitExitUninterruptibly(); + if (execution.attempt().abandoned()) { + execution.attempt().closeInvalidatedConnections(); + } + } + } finally { + worker.shutdownNow(); + abortWorker.shutdownNow(); + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + + private static boolean stop(ActiveExecution execution) { + JdbcMetadataMigrationAttempt attempt = execution.attempt(); + JdbcMetadataMigrationAttempt.Abandonment abandonment = attempt.abandon(); + if (!abandonment.abandoned()) { + if (attempt.abandoned()) { + attempt.abortConnections(); + } + return false; + } + execution.submitted().cancel(true); + attempt.abortConnections(); + if (!abandonment.workerStarted()) { + attempt.completeNeverStartedWorker(); + } + return true; + } + + private static MetadataMigrationException timeoutFailure() { + return new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT); + } + + private static ThreadPoolExecutor worker() { + ThreadPoolExecutor executor = new ThreadPoolExecutor( + 0, 1, 30, TimeUnit.SECONDS, new SynchronousQueue<>(), + Thread.ofPlatform().daemon(true).name("metadata-migration-copy", 0).factory(), + new ThreadPoolExecutor.AbortPolicy()); + executor.allowCoreThreadTimeOut(true); + return executor; + } + + private static ThreadPoolExecutor abortWorker() { + ThreadPoolExecutor executor = new ThreadPoolExecutor( + 0, 2, 30, TimeUnit.SECONDS, new SynchronousQueue<>(), + Thread.ofPlatform().daemon(true).name("metadata-migration-abort", 0).factory(), + new ThreadPoolExecutor.AbortPolicy()); + executor.allowCoreThreadTimeOut(true); + return executor; + } + + @FunctionalInterface + interface MigrationWork { + void migrate( + Connection source, + Connection target, + MetadataDatabaseKind targetKind, + Duration timeout, + MetadataMigrationProgressSink progress); + } + + private record ActiveExecution(JdbcMetadataMigrationAttempt attempt, Future submitted) { + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMigrationConnectionScope.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMigrationConnectionScope.java new file mode 100644 index 0000000000..ede916e8d7 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMigrationConnectionScope.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Objects; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; + +/** Owns temporary network deadlines and fail-closed disposal for one connection pair. */ +final class JdbcMigrationConnectionScope { + + private final Object lifecycleLock = new Object(); + private final Connection source; + private final Connection target; + private final JdbcMetadataMigrationDeadline deadline; + private final Executor networkExecutor; + private final ThreadPoolExecutor abortWorker; + private CountDownLatch abortCompleted; + private CountDownLatch closeCompleted; + private Error abortFailure; + private Error closeFailure; + private int sourceNetworkTimeout; + private int targetNetworkTimeout; + private boolean configured; + private boolean abortIssued; + private boolean closeIssued; + + JdbcMigrationConnectionScope( + Connection source, + Connection target, + JdbcMetadataMigrationDeadline deadline, + Executor networkExecutor, + ThreadPoolExecutor abortWorker) { + this.source = Objects.requireNonNull(source, "source"); + this.target = Objects.requireNonNull(target, "target"); + this.deadline = Objects.requireNonNull(deadline, "deadline"); + this.networkExecutor = Objects.requireNonNull(networkExecutor, "networkExecutor"); + this.abortWorker = Objects.requireNonNull(abortWorker, "abortWorker"); + } + + void configure() throws SQLException { + sourceNetworkTimeout = source.getNetworkTimeout(); + targetNetworkTimeout = target.getNetworkTimeout(); + source.setNetworkTimeout(networkExecutor, deadline.remainingMillis()); + target.setNetworkTimeout(networkExecutor, deadline.remainingMillis()); + configured = true; + } + + boolean restore() { + if (!configured) { + return true; + } + RestoreResult sourceResult = restore(source, sourceNetworkTimeout); + RestoreResult targetResult = restore(target, targetNetworkTimeout); + Error failure = targetResult.failure() == null ? sourceResult.failure() : targetResult.failure(); + if (failure != null) { + throw failure; + } + return sourceResult.restored() && targetResult.restored(); + } + + Error abort() { + CountDownLatch completion; + boolean start; + synchronized (lifecycleLock) { + start = !abortIssued; + if (start) { + abortIssued = true; + abortCompleted = new CountDownLatch(2); + } + completion = abortCompleted; + } + if (start) { + submitAbort(source, completion); + submitAbort(target, completion); + } + awaitUninterruptibly(completion); + synchronized (lifecycleLock) { + return abortFailure; + } + } + + Error closeInvalidated() { + CountDownLatch completion; + boolean start; + synchronized (lifecycleLock) { + start = !closeIssued; + if (start) { + closeIssued = true; + closeCompleted = new CountDownLatch(1); + } + completion = closeCompleted; + } + if (start) { + Error sourceFailure = close(source); + Error targetFailure = close(target); + synchronized (lifecycleLock) { + closeFailure = targetFailure == null ? sourceFailure : targetFailure; + } + completion.countDown(); + } else { + awaitUninterruptibly(completion); + } + synchronized (lifecycleLock) { + return closeFailure; + } + } + + private RestoreResult restore(Connection connection, int timeoutMillis) { + try { + connection.setNetworkTimeout(networkExecutor, timeoutMillis); + return new RestoreResult(true, null); + } catch (SQLException | RuntimeException failure) { + return new RestoreResult(false, null); + } catch (Error fatal) { + return new RestoreResult(false, fatal); + } + } + + private void submitAbort(Connection connection, CountDownLatch completion) { + try { + abortWorker.execute(() -> { + try { + connection.abort(networkExecutor); + } catch (SQLException | RuntimeException failure) { + // The stable primary outcome must not retain driver or endpoint details. + } catch (Error fatal) { + recordAbortFailure(fatal); + } finally { + completion.countDown(); + } + }); + } catch (RejectedExecutionException rejected) { + completion.countDown(); + } catch (Error fatal) { + recordAbortFailure(fatal); + completion.countDown(); + } + } + + private void recordAbortFailure(Error failure) { + synchronized (lifecycleLock) { + abortFailure = failure; + } + } + + private static Error close(Connection connection) { + try { + connection.close(); + return null; + } catch (SQLException | RuntimeException failure) { + return null; + } catch (Error fatal) { + return fatal; + } + } + + private static void awaitUninterruptibly(CountDownLatch latch) { + boolean interrupted = false; + while (latch.getCount() > 0) { + try { + latch.await(); + } catch (InterruptedException ignored) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + private record RestoreResult(boolean restored, Error failure) { + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationConnectionLifecycleTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationConnectionLifecycleTest.java new file mode 100644 index 0000000000..0c2132db75 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationConnectionLifecycleTest.java @@ -0,0 +1,287 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.SQLException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.mockito.ArgumentCaptor; + +@Timeout(15) +class JdbcMetadataMigrationConnectionLifecycleTest { + + private static final Duration TIMEOUT = Duration.ofSeconds(5); + private final List executors = new ArrayList<>(); + + @AfterEach + void closeExecutors() { + executors.forEach(JdbcMetadataMigrationExecutor::close); + } + + @Test + void configuresAndRestoresBothNetworkTimeoutsAroundSuccessfulCopy() throws Exception { + Connection source = connection(111); + Connection target = connection(222); + Executor networkExecutor = Runnable::run; + AtomicInteger calls = new AtomicInteger(); + JdbcMetadataMigrationExecutor executor = executor(networkExecutor, + (actualSource, actualTarget, kind, timeout, progress) -> { + assertThat(actualSource).isSameAs(source); + assertThat(actualTarget).isSameAs(target); + assertThat(kind).isEqualTo(MetadataDatabaseKind.POSTGRESQL); + assertThat(timeout).isPositive().isLessThanOrEqualTo(TIMEOUT); + calls.incrementAndGet(); + }); + + executor.execute(source, target, MetadataDatabaseKind.POSTGRESQL, TIMEOUT, + MetadataMigrationProgressSink.NO_OP); + + assertThat(calls).hasValue(1); + verify(source).getNetworkTimeout(); + verify(target).getNetworkTimeout(); + ArgumentCaptor sourceTimeouts = ArgumentCaptor.forClass(Integer.class); + verify(source, times(2)).setNetworkTimeout(eq(networkExecutor), sourceTimeouts.capture()); + assertThat(sourceTimeouts.getAllValues().getFirst()).isBetween(1, 5_000); + assertThat(sourceTimeouts.getAllValues().getLast()).isEqualTo(111); + ArgumentCaptor targetTimeouts = ArgumentCaptor.forClass(Integer.class); + verify(target, times(2)).setNetworkTimeout(eq(networkExecutor), targetTimeouts.capture()); + assertThat(targetTimeouts.getAllValues().getFirst()).isBetween(1, 5_000); + assertThat(targetTimeouts.getAllValues().getLast()).isEqualTo(222); + verify(source, never()).abort(any()); + verify(target, never()).abort(any()); + } + + @Test + void networkTimeoutConfigurationFailureAbortsAndClosesWithoutStartingCopy() throws Exception { + Connection source = connection(0); + Connection target = connection(0); + doThrow(new SQLException("private target path")).when(target).setNetworkTimeout(any(), anyInt()); + AtomicInteger calls = new AtomicInteger(); + JdbcMetadataMigrationExecutor executor = executor(Runnable::run, + (ignoredSource, ignoredTarget, kind, timeout, progress) -> calls.incrementAndGet()); + + assertThatThrownBy(() -> executor.execute(source, target, MetadataDatabaseKind.MYSQL, + TIMEOUT, MetadataMigrationProgressSink.NO_OP)) + .isInstanceOfSatisfying(MetadataMigrationException.class, failure -> { + assertThat(failure.code()).isEqualTo(MetadataMigrationErrorCode.TIMEOUT); + assertThat(failure).hasNoCause(); + assertThat(failure.getSuppressed()).isEmpty(); + assertThat(failure.getMessage()).doesNotContain("private target path"); + }); + + assertThat(calls).hasValue(0); + verifyInvalidated(source, target); + } + + @Test + void stableCopyFailureIsPreservedAfterNetworkTimeoutRestoration() throws Exception { + Connection source = connection(111); + Connection target = connection(222); + MetadataMigrationException original = new MetadataMigrationException(MetadataMigrationErrorCode.VERIFICATION); + JdbcMetadataMigrationExecutor executor = executor(Runnable::run, + (ignoredSource, ignoredTarget, kind, timeout, progress) -> { + throw original; + }); + + assertThatThrownBy(() -> executor.execute(source, target, MetadataDatabaseKind.POSTGRESQL, + TIMEOUT, MetadataMigrationProgressSink.NO_OP)) + .isSameAs(original); + + verify(source).setNetworkTimeout(any(), eq(111)); + verify(target).setNetworkTimeout(any(), eq(222)); + verify(source, never()).abort(any()); + verify(target, never()).abort(any()); + } + + @Test + void restorationFailureInvalidatesConnectionsWithoutChangingCommittedSuccess() throws Exception { + Connection source = connection(111); + Connection target = connection(222); + AtomicInteger sourceSets = new AtomicInteger(); + doAnswer(invocation -> { + if (sourceSets.incrementAndGet() == 2) { + throw new SQLException("private restore path"); + } + return null; + }).when(source).setNetworkTimeout(any(), anyInt()); + JdbcMetadataMigrationExecutor executor = executor(Runnable::run, + (ignoredSource, ignoredTarget, kind, timeout, progress) -> { }); + + executor.execute(source, target, MetadataDatabaseKind.MYSQL, + TIMEOUT, MetadataMigrationProgressSink.NO_OP); + + verifyInvalidated(source, target); + } + + @Test + void deadlineAbandonsWorkerBlockedWhileRestoringConnectionState() throws Exception { + Connection source = connection(111); + Connection target = connection(222); + AtomicInteger sourceSets = new AtomicInteger(); + CountDownLatch restoreEntered = new CountDownLatch(1); + CountDownLatch releaseRestore = new CountDownLatch(1); + CountDownLatch abortCalled = new CountDownLatch(2); + doAnswer(invocation -> { + if (sourceSets.incrementAndGet() == 2) { + restoreEntered.countDown(); + awaitIgnoringInterrupt(releaseRestore); + } + return null; + }).when(source).setNetworkTimeout(any(), anyInt()); + doAnswer(invocation -> { + abortCalled.countDown(); + return null; + }).when(source).abort(any()); + doAnswer(invocation -> { + abortCalled.countDown(); + return null; + }).when(target).abort(any()); + JdbcMetadataMigrationExecutor executor = executor(Runnable::run, + (ignoredSource, ignoredTarget, kind, timeout, progress) -> { }); + try (ExecutorService caller = Executors.newSingleThreadExecutor()) { + Future result = caller.submit(() -> failureCode(() -> executor.execute( + source, target, MetadataDatabaseKind.MYSQL, Duration.ofMillis(20), + MetadataMigrationProgressSink.NO_OP))); + try { + assertThat(restoreEntered.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(abortCalled.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(result.isDone()).isFalse(); + } finally { + releaseRestore.countDown(); + } + assertThat(result.get(5, TimeUnit.SECONDS)).isEqualTo(MetadataMigrationErrorCode.TIMEOUT); + } + } + + @Test + void fatalConfigurationErrorStillExitsAndFailsBothConnectionsClosed() throws Exception { + Connection source = connection(0); + Connection target = connection(0); + AssertionError fatal = new AssertionError("fatal network timeout read"); + doThrow(fatal).when(source).getNetworkTimeout(); + AtomicInteger calls = new AtomicInteger(); + JdbcMetadataMigrationExecutor executor = executor(Runnable::run, + (ignoredSource, ignoredTarget, kind, timeout, progress) -> calls.incrementAndGet()); + + assertThatThrownBy(() -> executor.execute(source, target, MetadataDatabaseKind.MYSQL, + TIMEOUT, MetadataMigrationProgressSink.NO_OP)) + .isSameAs(fatal); + + assertThat(calls).hasValue(0); + verifyInvalidated(source, target); + } + + @Test + void fatalRestorationErrorCannotBypassTheWorkerExitProof() throws Exception { + Connection source = connection(111); + Connection target = connection(222); + AssertionError fatal = new AssertionError("fatal network timeout restore"); + AtomicInteger sourceSets = new AtomicInteger(); + doAnswer(invocation -> { + if (sourceSets.incrementAndGet() == 2) { + throw fatal; + } + return null; + }).when(source).setNetworkTimeout(any(), anyInt()); + JdbcMetadataMigrationExecutor executor = executor(Runnable::run, + (ignoredSource, ignoredTarget, kind, timeout, progress) -> { }); + + assertThatThrownBy(() -> executor.execute(source, target, MetadataDatabaseKind.POSTGRESQL, + TIMEOUT, MetadataMigrationProgressSink.NO_OP)) + .isSameAs(fatal); + + verifyInvalidated(source, target); + } + + private static void verifyInvalidated(Connection source, Connection target) throws Exception { + verify(source).abort(any()); + verify(target).abort(any()); + verify(source).close(); + verify(target).close(); + } + + private static Connection connection(int networkTimeout) throws Exception { + Connection connection = mock(Connection.class); + when(connection.getNetworkTimeout()).thenReturn(networkTimeout); + return connection; + } + + private JdbcMetadataMigrationExecutor executor( + Executor networkExecutor, JdbcMetadataMigrationExecutor.MigrationWork work) { + JdbcMetadataMigrationExecutor executor = + new JdbcMetadataMigrationExecutor(worker(), networkExecutor, work); + executors.add(executor); + return executor; + } + + private static ThreadPoolExecutor worker() { + ThreadPoolExecutor worker = new ThreadPoolExecutor( + 0, 1, 30, TimeUnit.SECONDS, new SynchronousQueue<>(), + Thread.ofPlatform().daemon(true).name("metadata-connection-test", 0).factory(), + new ThreadPoolExecutor.AbortPolicy()); + worker.allowCoreThreadTimeOut(true); + return worker; + } + + private static MetadataMigrationErrorCode failureCode(ThrowingAction action) { + try { + action.run(); + throw new AssertionError("Expected migration failure"); + } catch (MetadataMigrationException failure) { + assertThat(failure).hasNoCause(); + return failure.code(); + } + } + + private static void awaitIgnoringInterrupt(CountDownLatch latch) { + boolean interrupted = false; + while (latch.getCount() > 0) { + try { + latch.await(); + } catch (InterruptedException ignored) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + @FunctionalInterface + private interface ThrowingAction { + void run(); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationDeadlineTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationDeadlineTest.java new file mode 100644 index 0000000000..c9562c636e --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationDeadlineTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Duration; +import java.util.concurrent.atomic.AtomicLong; +import org.junit.jupiter.api.Test; + +class JdbcMetadataMigrationDeadlineTest { + + @Test + void negativeMonotonicTickerDoesNotOverflowTheDeadline() { + AtomicLong ticker = new AtomicLong(-100); + JdbcMetadataMigrationDeadline deadline = + JdbcMetadataMigrationDeadline.start(Duration.ofNanos(50), ticker::get); + + assertThat(deadline.remainingNanos()).isEqualTo(50); + ticker.set(-75); + assertThat(deadline.remainingNanos()).isEqualTo(25); + ticker.set(-50); + assertThat(deadline.remainingNanos()).isZero(); + } + + @Test + void remainingBudgetSaturatesInsteadOfOverflowingAcrossTheSignedBoundary() { + AtomicLong ticker = new AtomicLong(-10); + JdbcMetadataMigrationDeadline deadline = + JdbcMetadataMigrationDeadline.start(Duration.ofNanos(Long.MAX_VALUE), ticker::get); + + assertThat(deadline.remainingNanos()).isEqualTo(Long.MAX_VALUE); + } + + @Test + void elapsedTimeRemainsExactWhenTheTickerCrossesZero() { + AtomicLong ticker = new AtomicLong(-10); + JdbcMetadataMigrationDeadline deadline = + JdbcMetadataMigrationDeadline.start(Duration.ofNanos(20), ticker::get); + + ticker.set(-5); + assertThat(deadline.remainingNanos()).isEqualTo(15); + ticker.set(5); + assertThat(deadline.remainingNanos()).isEqualTo(5); + ticker.set(10); + assertThat(deadline.remainingNanos()).isZero(); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationExecutorLifecycleTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationExecutorLifecycleTest.java new file mode 100644 index 0000000000..5434c4271d --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationExecutorLifecycleTest.java @@ -0,0 +1,380 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +@Timeout(15) +class JdbcMetadataMigrationExecutorLifecycleTest { + + private static final Duration TIMEOUT = Duration.ofSeconds(5); + + @Test + void concurrentExecutionCannotReplaceTheActiveAttempt() throws Exception { + Connection source = connection(); + Connection target = connection(); + Connection secondSource = connection(); + Connection secondTarget = connection(); + CountDownLatch workStarted = new CountDownLatch(1); + CountDownLatch releaseWork = new CountDownLatch(1); + JdbcMetadataMigrationExecutor executor = executor( + (ignoredSource, ignoredTarget, kind, timeout, progress) -> { + workStarted.countDown(); + awaitIgnoringInterrupt(releaseWork); + }); + try (executor; ExecutorService caller = Executors.newSingleThreadExecutor()) { + Future first = caller.submit(() -> executor.execute(source, target, + MetadataDatabaseKind.MYSQL, TIMEOUT, MetadataMigrationProgressSink.NO_OP)); + try { + assertThat(workStarted.await(5, TimeUnit.SECONDS)).isTrue(); + assertThatThrownBy(() -> executor.execute(secondSource, secondTarget, + MetadataDatabaseKind.POSTGRESQL, TIMEOUT, + MetadataMigrationProgressSink.NO_OP)) + .isInstanceOfSatisfying(MetadataMigrationException.class, + failure -> assertThat(failure.code()) + .isEqualTo(MetadataMigrationErrorCode.TIMEOUT)); + verify(secondSource, never()).getNetworkTimeout(); + verify(secondTarget, never()).getNetworkTimeout(); + } finally { + releaseWork.countDown(); + } + first.get(5, TimeUnit.SECONDS); + } + } + + @Test + void closeWaitsUntilTheActiveWorkerHasActuallyExited() throws Exception { + Connection source = connection(); + Connection target = connection(); + CountDownLatch workStarted = new CountDownLatch(1); + CountDownLatch releaseWork = new CountDownLatch(1); + CountDownLatch abortCalled = new CountDownLatch(2); + doAnswer(invocation -> { + abortCalled.countDown(); + return null; + }).when(source).abort(any()); + doAnswer(invocation -> { + abortCalled.countDown(); + return null; + }).when(target).abort(any()); + JdbcMetadataMigrationExecutor executor = executor( + (ignoredSource, ignoredTarget, kind, timeout, progress) -> { + workStarted.countDown(); + awaitIgnoringInterrupt(releaseWork); + }); + try (ExecutorService caller = Executors.newSingleThreadExecutor(); + ExecutorService closer = Executors.newSingleThreadExecutor()) { + Future result = caller.submit(() -> failureCode(() -> executor.execute( + source, target, MetadataDatabaseKind.MYSQL, TIMEOUT, + MetadataMigrationProgressSink.NO_OP))); + Future closeResult = null; + try { + assertThat(workStarted.await(5, TimeUnit.SECONDS)).isTrue(); + closeResult = closer.submit(executor::close); + assertThat(abortCalled.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(closeResult.isDone()).isFalse(); + } finally { + releaseWork.countDown(); + } + if (closeResult != null) { + closeResult.get(5, TimeUnit.SECONDS); + } + assertThat(result.get(5, TimeUnit.SECONDS)).isEqualTo(MetadataMigrationErrorCode.TIMEOUT); + } finally { + releaseWork.countDown(); + executor.close(); + } + } + + @Test + void concurrentCloseJoinsAnAlreadyStartedAbortAndInvalidation() throws Exception { + Connection source = connection(); + Connection target = connection(); + CountDownLatch workStarted = new CountDownLatch(1); + CountDownLatch workExited = new CountDownLatch(1); + CountDownLatch releaseWork = new CountDownLatch(1); + CountDownLatch abortEntered = new CountDownLatch(2); + CountDownLatch releaseAbort = new CountDownLatch(1); + CountDownLatch closeStarted = new CountDownLatch(1); + CountDownLatch closeReturned = new CountDownLatch(1); + doAnswer(invocation -> { + abortEntered.countDown(); + awaitIgnoringInterrupt(releaseAbort); + return null; + }).when(source).abort(any()); + doAnswer(invocation -> { + abortEntered.countDown(); + awaitIgnoringInterrupt(releaseAbort); + return null; + }).when(target).abort(any()); + JdbcMetadataMigrationExecutor executor = executor( + (ignoredSource, ignoredTarget, kind, timeout, progress) -> { + workStarted.countDown(); + try { + releaseWork.await(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } finally { + workExited.countDown(); + } + }); + try (ExecutorService caller = Executors.newSingleThreadExecutor(); + ExecutorService closer = Executors.newSingleThreadExecutor()) { + Future result = caller.submit(() -> failureCode(() -> executor.execute( + source, target, MetadataDatabaseKind.MYSQL, Duration.ofMillis(20), + MetadataMigrationProgressSink.NO_OP))); + Future closeResult = null; + try { + assertThat(workStarted.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(abortEntered.await(5, TimeUnit.SECONDS)).isTrue(); + closeResult = closer.submit(() -> { + closeStarted.countDown(); + try { + executor.close(); + } finally { + closeReturned.countDown(); + } + }); + assertThat(closeStarted.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(workExited.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(closeReturned.await(1, TimeUnit.SECONDS)).isFalse(); + } finally { + releaseWork.countDown(); + releaseAbort.countDown(); + } + if (closeResult != null) { + closeResult.get(5, TimeUnit.SECONDS); + } + assertThat(result.get(5, TimeUnit.SECONDS)).isEqualTo(MetadataMigrationErrorCode.TIMEOUT); + verify(source, times(1)).close(); + verify(target, times(1)).close(); + } finally { + releaseWork.countDown(); + releaseAbort.countDown(); + executor.close(); + } + } + + @Test + void copyWorkerCloseFailsFastInsteadOfWaitingForItself() throws Exception { + Connection source = connection(); + Connection target = connection(); + AtomicReference executorRef = new AtomicReference<>(); + AtomicReference closeFailure = new AtomicReference<>(); + JdbcMetadataMigrationExecutor executor = executor( + (ignoredSource, ignoredTarget, kind, timeout, progress) -> { + try { + executorRef.get().close(); + } catch (RuntimeException failure) { + closeFailure.set(failure); + throw failure; + } + }); + executorRef.set(executor); + + try (executor) { + assertThatThrownBy(() -> executor.execute(source, target, MetadataDatabaseKind.MYSQL, + TIMEOUT, MetadataMigrationProgressSink.NO_OP)) + .isInstanceOfSatisfying(MetadataMigrationException.class, + failure -> assertThat(failure.code()) + .isEqualTo(MetadataMigrationErrorCode.COPY)); + assertThat(closeFailure.get()).isInstanceOf(IllegalStateException.class); + } + + verify(source, never()).close(); + verify(target, never()).close(); + } + + @Test + void progressCallbackCloseFailsFastInsteadOfWaitingForItsWorker() throws Exception { + Connection source = connection(); + Connection target = connection(); + AtomicReference executorRef = new AtomicReference<>(); + AtomicReference closeFailure = new AtomicReference<>(); + JdbcMetadataMigrationExecutor executor = executor( + (ignoredSource, ignoredTarget, kind, timeout, progress) -> + progress.report(MetadataMigrationStage.COPYING, 50)); + executorRef.set(executor); + MetadataMigrationProgressSink progress = (stage, percent) -> { + try { + executorRef.get().close(); + } catch (RuntimeException failure) { + closeFailure.set(failure); + throw failure; + } + }; + + try (executor) { + assertThatThrownBy(() -> executor.execute(source, target, MetadataDatabaseKind.MYSQL, + TIMEOUT, progress)) + .isInstanceOfSatisfying(MetadataMigrationException.class, + failure -> assertThat(failure.code()) + .isEqualTo(MetadataMigrationErrorCode.COPY)); + assertThat(closeFailure.get()).isInstanceOf(IllegalStateException.class); + } + + verify(source, never()).close(); + verify(target, never()).close(); + } + + @Test + void rejectedWorkerDoesNotTouchCallerConnections() throws Exception { + Connection source = connection(); + Connection target = connection(); + ThreadPoolExecutor rejectedWorker = worker("metadata-copy-rejected"); + rejectedWorker.shutdownNow(); + JdbcMetadataMigrationExecutor executor = new JdbcMetadataMigrationExecutor( + rejectedWorker, Runnable::run, + (ignoredSource, ignoredTarget, kind, timeout, progress) -> { }); + try (executor) { + assertThatThrownBy(() -> executor.execute(source, target, MetadataDatabaseKind.MYSQL, + TIMEOUT, MetadataMigrationProgressSink.NO_OP)) + .isInstanceOfSatisfying(MetadataMigrationException.class, + failure -> assertThat(failure.code()) + .isEqualTo(MetadataMigrationErrorCode.TIMEOUT)); + } + + verify(source, never()).getNetworkTimeout(); + verify(target, never()).getNetworkTimeout(); + } + + @Test + void fatalWorkerSubmissionClearsThePublishedAttemptWithoutTouchingConnections() throws Exception { + Connection source = connection(); + Connection target = connection(); + AssertionError fatal = new AssertionError("fatal worker submission"); + ThreadPoolExecutor failedWorker = throwingWorker(fatal); + JdbcMetadataMigrationExecutor executor = new JdbcMetadataMigrationExecutor( + failedWorker, Runnable::run, + (ignoredSource, ignoredTarget, kind, timeout, progress) -> { }); + + try (executor) { + assertThatThrownBy(() -> executor.execute(source, target, MetadataDatabaseKind.MYSQL, + TIMEOUT, MetadataMigrationProgressSink.NO_OP)) + .isSameAs(fatal); + } + + verify(source, never()).getNetworkTimeout(); + verify(source, never()).abort(any()); + verify(source, never()).close(); + verify(target, never()).getNetworkTimeout(); + verify(target, never()).abort(any()); + verify(target, never()).close(); + } + + @Test + void runtimeWorkerSubmissionIsRedactedAndClearsThePublishedAttempt() throws Exception { + Connection source = connection(); + Connection target = connection(); + ThreadPoolExecutor failedWorker = throwingWorker(new IllegalStateException("private worker detail")); + JdbcMetadataMigrationExecutor executor = new JdbcMetadataMigrationExecutor( + failedWorker, Runnable::run, + (ignoredSource, ignoredTarget, kind, timeout, progress) -> { }); + + try (executor) { + assertThatThrownBy(() -> executor.execute(source, target, MetadataDatabaseKind.MYSQL, + TIMEOUT, MetadataMigrationProgressSink.NO_OP)) + .isInstanceOfSatisfying(MetadataMigrationException.class, failure -> { + assertThat(failure.code()).isEqualTo(MetadataMigrationErrorCode.TIMEOUT); + assertThat(failure).hasNoCause(); + assertThat(failure.getMessage()).doesNotContain("private worker detail"); + }); + } + + verify(source, never()).abort(any()); + verify(source, never()).close(); + verify(target, never()).abort(any()); + verify(target, never()).close(); + } + + private static JdbcMetadataMigrationExecutor executor( + JdbcMetadataMigrationExecutor.MigrationWork work) { + return new JdbcMetadataMigrationExecutor(worker("metadata-copy-lifecycle"), Runnable::run, work); + } + + private static ThreadPoolExecutor worker(String name) { + ThreadPoolExecutor worker = new ThreadPoolExecutor( + 0, 2, 30, TimeUnit.SECONDS, new SynchronousQueue<>(), + Thread.ofPlatform().daemon(true).name(name, 0).factory(), + new ThreadPoolExecutor.AbortPolicy()); + worker.allowCoreThreadTimeOut(true); + return worker; + } + + private static ThreadPoolExecutor throwingWorker(Throwable failure) { + return new ThreadPoolExecutor( + 0, 1, 30, TimeUnit.SECONDS, new SynchronousQueue<>(), + Thread.ofPlatform().daemon(true).name("metadata-copy-submission-failure", 0).factory(), + new ThreadPoolExecutor.AbortPolicy()) { + @Override + public void execute(Runnable command) { + if (failure instanceof Error error) { + throw error; + } + throw (RuntimeException) failure; + } + }; + } + + private static Connection connection() throws Exception { + Connection connection = mock(Connection.class); + when(connection.getNetworkTimeout()).thenReturn(0); + return connection; + } + + private static MetadataMigrationErrorCode failureCode(ThrowingAction action) { + try { + action.run(); + throw new AssertionError("Expected migration failure"); + } catch (MetadataMigrationException failure) { + assertThat(failure).hasNoCause(); + return failure.code(); + } + } + + private static void awaitIgnoringInterrupt(CountDownLatch latch) { + boolean interrupted = false; + while (latch.getCount() > 0) { + try { + latch.await(); + } catch (InterruptedException ignored) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + @FunctionalInterface + private interface ThrowingAction { + void run(); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationExecutorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationExecutorTest.java new file mode 100644 index 0000000000..85aaabddf1 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationExecutorTest.java @@ -0,0 +1,421 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.SQLException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +@Timeout(15) +class JdbcMetadataMigrationExecutorTest { + + private static final Duration TIMEOUT = Duration.ofSeconds(5); + private final List executors = new ArrayList<>(); + + @AfterEach + void closeExecutors() { + executors.forEach(JdbcMetadataMigrationExecutor::close); + } + + @Test + void timeoutAbortsAndCancelsButDoesNotReturnUntilWorkerReallyExits() throws Exception { + Connection source = connection(0); + Connection target = connection(0); + CountDownLatch workStarted = new CountDownLatch(1); + CountDownLatch releaseWork = new CountDownLatch(1); + CountDownLatch abortCalled = new CountDownLatch(2); + AtomicInteger forwardedProgress = new AtomicInteger(); + doAnswer(invocation -> { + abortCalled.countDown(); + return null; + }).when(source).abort(any()); + doAnswer(invocation -> { + abortCalled.countDown(); + return null; + }).when(target).abort(any()); + JdbcMetadataMigrationExecutor executor = executor(Runnable::run, + (ignoredSource, ignoredTarget, kind, timeout, progress) -> { + workStarted.countDown(); + awaitIgnoringInterrupt(releaseWork); + progress.report(MetadataMigrationStage.COPYING, 50); + }); + try (ExecutorService caller = Executors.newSingleThreadExecutor()) { + Future result = caller.submit(() -> failureCode(() -> executor.execute( + source, target, MetadataDatabaseKind.MYSQL, Duration.ofMillis(20), + (stage, percent) -> forwardedProgress.incrementAndGet()))); + try { + assertThat(workStarted.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(abortCalled.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(result.isDone()).isFalse(); + } finally { + releaseWork.countDown(); + } + assertThat(result.get(5, TimeUnit.SECONDS)).isEqualTo(MetadataMigrationErrorCode.TIMEOUT); + assertThat(forwardedProgress).hasValue(0); + verify(source).close(); + verify(target).close(); + } + } + + @Test + void interruptionWaitsForWorkerExitAndRestoresCallerInterruptFlag() throws Exception { + Connection source = connection(0); + Connection target = connection(0); + CountDownLatch workStarted = new CountDownLatch(1); + CountDownLatch releaseWork = new CountDownLatch(1); + CountDownLatch abortCalled = new CountDownLatch(2); + doAnswer(invocation -> { + abortCalled.countDown(); + return null; + }).when(source).abort(any()); + doAnswer(invocation -> { + abortCalled.countDown(); + return null; + }).when(target).abort(any()); + JdbcMetadataMigrationExecutor executor = executor(Runnable::run, + (ignoredSource, ignoredTarget, kind, timeout, progress) -> { + workStarted.countDown(); + awaitIgnoringInterrupt(releaseWork); + }); + AtomicReference code = new AtomicReference<>(); + AtomicBoolean interrupted = new AtomicBoolean(); + Thread caller = Thread.ofPlatform().start(() -> { + code.set(failureCode(() -> executor.execute(source, target, MetadataDatabaseKind.MYSQL, + TIMEOUT, MetadataMigrationProgressSink.NO_OP))); + interrupted.set(Thread.currentThread().isInterrupted()); + }); + + try { + assertThat(workStarted.await(5, TimeUnit.SECONDS)).isTrue(); + caller.interrupt(); + assertThat(abortCalled.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(caller.isAlive()).isTrue(); + } finally { + releaseWork.countDown(); + } + caller.join(5_000); + + assertThat(caller.isAlive()).isFalse(); + assertThat(code).hasValue(MetadataMigrationErrorCode.TIMEOUT); + assertThat(interrupted).isTrue(); + } + + @Test + void abortFailureNeverEscapesThroughTimeoutFailure() throws Exception { + Connection source = connection(0); + Connection target = connection(0); + CountDownLatch workStarted = new CountDownLatch(1); + CountDownLatch releaseWork = new CountDownLatch(1); + doThrow(new SQLException("private abort provider")).when(source).abort(any()); + doAnswer(invocation -> { + releaseWork.countDown(); + return null; + }).when(target).abort(any()); + JdbcMetadataMigrationExecutor executor = executor(Runnable::run, + (ignoredSource, ignoredTarget, kind, timeout, progress) -> { + workStarted.countDown(); + awaitIgnoringInterrupt(releaseWork); + }); + + try (ExecutorService caller = Executors.newSingleThreadExecutor()) { + Future result = caller.submit(() -> migrationFailure(() -> executor.execute( + source, target, MetadataDatabaseKind.MYSQL, Duration.ofMillis(20), + MetadataMigrationProgressSink.NO_OP))); + try { + assertThat(workStarted.await(5, TimeUnit.SECONDS)).isTrue(); + MetadataMigrationException failure = result.get(5, TimeUnit.SECONDS); + assertThat(failure.code()).isEqualTo(MetadataMigrationErrorCode.TIMEOUT); + assertThat(failure).hasNoCause(); + assertThat(failure.getSuppressed()).isEmpty(); + assertThat(failure.getMessage()).doesNotContain("private abort provider"); + } finally { + releaseWork.countDown(); + } + } + } + + @Test + void fatalWorkerErrorIsRethrownOnlyAfterConnectionStateIsRestored() throws Exception { + Connection source = connection(111); + Connection target = connection(222); + AssertionError fatal = new AssertionError("fatal copy failure"); + JdbcMetadataMigrationExecutor executor = executor(Runnable::run, + (ignoredSource, ignoredTarget, kind, timeout, progress) -> { + throw fatal; + }); + + assertThatThrownBy(() -> executor.execute(source, target, MetadataDatabaseKind.MYSQL, + TIMEOUT, MetadataMigrationProgressSink.NO_OP)) + .isSameAs(fatal); + + verify(source).setNetworkTimeout(any(), eq(111)); + verify(target).setNetworkTimeout(any(), eq(222)); + } + + @Test + void commitOutcomeUnknownOutranksAnOuterTimeout() throws Exception { + Connection source = connection(0); + Connection target = connection(0); + CountDownLatch workStarted = new CountDownLatch(1); + CountDownLatch abortCalled = new CountDownLatch(2); + doAnswer(invocation -> { + abortCalled.countDown(); + return null; + }).when(source).abort(any()); + doAnswer(invocation -> { + abortCalled.countDown(); + return null; + }).when(target).abort(any()); + JdbcMetadataMigrationExecutor executor = executor(Runnable::run, + (ignoredSource, ignoredTarget, kind, timeout, progress) -> { + workStarted.countDown(); + awaitIgnoringInterrupt(abortCalled); + throw new MetadataMigrationException(MetadataMigrationErrorCode.COMMIT_OUTCOME_UNKNOWN); + }); + + try (ExecutorService caller = Executors.newSingleThreadExecutor()) { + Future result = caller.submit(() -> failureCode(() -> executor.execute( + source, target, MetadataDatabaseKind.MYSQL, Duration.ofMillis(20), + MetadataMigrationProgressSink.NO_OP))); + try { + assertThat(workStarted.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(result.get(5, TimeUnit.SECONDS)) + .isEqualTo(MetadataMigrationErrorCode.COMMIT_OUTCOME_UNKNOWN); + } finally { + abortCalled.countDown(); + abortCalled.countDown(); + } + } + } + + @Test + void abortErrorOutranksLaterCommitOutcomeUnknown() throws Exception { + Connection source = connection(0); + Connection target = connection(0); + AssertionError abortFatal = new AssertionError("fatal abort failure"); + doThrow(abortFatal).when(source).abort(any()); + ThreadPoolExecutor abortWorker = worker(); + JdbcMetadataMigrationDeadline deadline = JdbcMetadataMigrationDeadline.start(TIMEOUT, System::nanoTime); + JdbcMigrationConnectionScope connections = new JdbcMigrationConnectionScope( + source, target, deadline, Runnable::run, abortWorker); + CountDownLatch workStarted = new CountDownLatch(1); + CountDownLatch releaseWork = new CountDownLatch(1); + JdbcMetadataMigrationAttempt attempt = new JdbcMetadataMigrationAttempt( + source, + target, + MetadataDatabaseKind.MYSQL, + deadline, + connections, + (ignoredSource, ignoredTarget, kind, timeout, progress) -> { + workStarted.countDown(); + awaitIgnoringInterrupt(releaseWork); + throw new MetadataMigrationException(MetadataMigrationErrorCode.COMMIT_OUTCOME_UNKNOWN); + }, + MetadataMigrationProgressSink.NO_OP); + Thread copyWorker = Thread.ofPlatform().start(attempt::run); + + try { + try { + assertThat(workStarted.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(attempt.abandon().abandoned()).isTrue(); + attempt.abortConnections(); + } finally { + releaseWork.countDown(); + } + copyWorker.join(5_000); + assertThat(copyWorker.isAlive()).isFalse(); + assertThatThrownBy(attempt::rethrowFatalOrOutcomeUnknown).isSameAs(abortFatal); + } finally { + releaseWork.countDown(); + copyWorker.join(5_000); + abortWorker.shutdownNow(); + } + } + + @Test + void blockedProgressSinkCannotPreventDeadlineAbort() throws Exception { + Connection source = connection(0); + Connection target = connection(0); + CountDownLatch progressEntered = new CountDownLatch(1); + CountDownLatch releaseProgress = new CountDownLatch(1); + CountDownLatch abortCalled = new CountDownLatch(2); + doAnswer(invocation -> { + abortCalled.countDown(); + return null; + }).when(source).abort(any()); + doAnswer(invocation -> { + abortCalled.countDown(); + return null; + }).when(target).abort(any()); + JdbcMetadataMigrationExecutor executor = executor(Runnable::run, + (ignoredSource, ignoredTarget, kind, timeout, progress) -> + progress.report(MetadataMigrationStage.COPYING, 50)); + try (ExecutorService caller = Executors.newSingleThreadExecutor()) { + Future result = caller.submit(() -> failureCode(() -> executor.execute( + source, target, MetadataDatabaseKind.MYSQL, Duration.ofMillis(20), (stage, percent) -> { + progressEntered.countDown(); + awaitIgnoringInterrupt(releaseProgress); + }))); + try { + assertThat(progressEntered.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(abortCalled.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(result.isDone()).isFalse(); + } finally { + releaseProgress.countDown(); + } + assertThat(result.get(5, TimeUnit.SECONDS)).isEqualTo(MetadataMigrationErrorCode.TIMEOUT); + } + } + + @Test + void oneBlockingAbortCannotPreventTheOtherConnectionAbort() throws Exception { + Connection source = connection(0); + Connection target = connection(0); + CountDownLatch workStarted = new CountDownLatch(1); + CountDownLatch sourceAbortEntered = new CountDownLatch(1); + CountDownLatch targetAbortCalled = new CountDownLatch(1); + CountDownLatch releaseSourceAbort = new CountDownLatch(1); + CountDownLatch releaseWork = new CountDownLatch(1); + doAnswer(invocation -> { + sourceAbortEntered.countDown(); + awaitIgnoringInterrupt(releaseSourceAbort); + return null; + }).when(source).abort(any()); + doAnswer(invocation -> { + targetAbortCalled.countDown(); + releaseWork.countDown(); + return null; + }).when(target).abort(any()); + JdbcMetadataMigrationExecutor executor = executor(Runnable::run, + (ignoredSource, ignoredTarget, kind, timeout, progress) -> { + workStarted.countDown(); + awaitIgnoringInterrupt(releaseWork); + }); + try (ExecutorService caller = Executors.newSingleThreadExecutor()) { + Future result = caller.submit(() -> failureCode(() -> executor.execute( + source, target, MetadataDatabaseKind.MYSQL, Duration.ofMillis(20), + MetadataMigrationProgressSink.NO_OP))); + try { + assertThat(workStarted.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(sourceAbortEntered.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(targetAbortCalled.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(result.isDone()).isFalse(); + } finally { + releaseSourceAbort.countDown(); + releaseWork.countDown(); + } + assertThat(result.get(5, TimeUnit.SECONDS)).isEqualTo(MetadataMigrationErrorCode.TIMEOUT); + } + } + + @Test + void closedExecutorRejectsWithoutTouchingCallerConnections() throws Exception { + Connection source = connection(0); + Connection target = connection(0); + JdbcMetadataMigrationExecutor executor = executor(Runnable::run, + (ignoredSource, ignoredTarget, kind, timeout, progress) -> { }); + executor.close(); + + assertThatThrownBy(() -> executor.execute(source, target, MetadataDatabaseKind.MYSQL, + TIMEOUT, MetadataMigrationProgressSink.NO_OP)) + .isInstanceOfSatisfying(MetadataMigrationException.class, + failure -> assertThat(failure.code()).isEqualTo(MetadataMigrationErrorCode.TIMEOUT)); + + verify(source, never()).getNetworkTimeout(); + verify(target, never()).getNetworkTimeout(); + } + + private static Connection connection(int networkTimeout) throws Exception { + Connection connection = mock(Connection.class); + when(connection.getNetworkTimeout()).thenReturn(networkTimeout); + return connection; + } + + private JdbcMetadataMigrationExecutor executor( + Executor networkExecutor, JdbcMetadataMigrationExecutor.MigrationWork work) { + JdbcMetadataMigrationExecutor executor = + new JdbcMetadataMigrationExecutor(worker(), networkExecutor, work); + executors.add(executor); + return executor; + } + + private static ThreadPoolExecutor worker() { + ThreadPoolExecutor worker = new ThreadPoolExecutor( + 0, 1, 30, TimeUnit.SECONDS, new SynchronousQueue<>(), + Thread.ofPlatform().daemon(true).name("metadata-copy-test", 0).factory(), + new ThreadPoolExecutor.AbortPolicy()); + worker.allowCoreThreadTimeOut(true); + return worker; + } + + private static MetadataMigrationErrorCode failureCode(ThrowingAction action) { + try { + action.run(); + throw new AssertionError("Expected migration failure"); + } catch (MetadataMigrationException failure) { + assertThat(failure).hasNoCause(); + return failure.code(); + } + } + + private static MetadataMigrationException migrationFailure(ThrowingAction action) { + try { + action.run(); + throw new AssertionError("Expected migration failure"); + } catch (MetadataMigrationException failure) { + return failure; + } + } + + private static void awaitIgnoringInterrupt(CountDownLatch latch) { + boolean interrupted = false; + while (latch.getCount() > 0) { + try { + latch.await(); + } catch (InterruptedException ignored) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + @FunctionalInterface + private interface ThrowingAction { + void run(); + } +} From 7dffb24c4b951ac7202eaf7421669fc2bb0112bb Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 08:52:50 +0800 Subject: [PATCH 45/71] Compose metadata copy maintenance --- .../CompositeMigrationMaintenanceLease.java | 26 ++ .../maintenance/EmbeddedH2SourceGuard.java | 35 +- .../MigrationMaintenanceException.java | 6 +- .../MigrationMaintenanceLease.java | 3 + .../maintenance/MigrationSourceAction.java | 22 ++ .../maintenance/MigrationSourceLease.java | 3 + .../JdbcMetadataMigrationExecutor.java | 13 +- .../MetadataCopyExecutionCoordinator.java | 194 ++++++++++ .../setup/workflow/MetadataCopyOutcome.java | 82 ++++ .../MetadataCopyReleaseRequiredException.java | 47 +++ ...tMigrationMaintenanceOrchestratorTest.java | 90 +++++ .../EmbeddedH2SourceGuardTest.java | 103 +++++ .../MetadataCopyExecutionCoordinatorTest.java | 365 ++++++++++++++++++ 13 files changed, 980 insertions(+), 9 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationSourceAction.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataCopyExecutionCoordinator.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataCopyOutcome.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataCopyReleaseRequiredException.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataCopyExecutionCoordinatorTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/CompositeMigrationMaintenanceLease.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/CompositeMigrationMaintenanceLease.java index b89ae13958..5888f913b6 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/CompositeMigrationMaintenanceLease.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/CompositeMigrationMaintenanceLease.java @@ -21,6 +21,8 @@ final class CompositeMigrationMaintenanceLease implements MigrationMaintenanceLe private boolean producerReleased; private boolean sourceReleased; private boolean authorityReleased; + private boolean sourceCallbackActive; + private Thread sourceCallbackOwner; CompositeMigrationMaintenanceLease( DeploymentSingletonLease authorityLease, @@ -35,8 +37,32 @@ final class CompositeMigrationMaintenanceLease implements MigrationMaintenanceLe this.reservationRelease = reservationRelease; } + @Override + public synchronized void withSourceConnection(MigrationSourceAction action) { + if (action == null) { + throw MigrationMaintenanceException.invalidRequest(); + } + if (writeReleased || producerReleased || sourceReleased || authorityReleased) { + throw MigrationMaintenanceException.operationConflict(); + } + if (sourceCallbackActive) { + throw MigrationMaintenanceException.operationConflict(); + } + sourceCallbackActive = true; + sourceCallbackOwner = Thread.currentThread(); + try { + sourceLease.withConnection(action); + } finally { + sourceCallbackOwner = null; + sourceCallbackActive = false; + } + } + @Override public synchronized void close() { + if (sourceCallbackActive && sourceCallbackOwner == Thread.currentThread()) { + throw MigrationMaintenanceException.operationConflict(); + } try { releaseInOrder(); } catch (MigrationMaintenanceException exception) { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/EmbeddedH2SourceGuard.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/EmbeddedH2SourceGuard.java index 9fc057a20e..b343203062 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/EmbeddedH2SourceGuard.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/EmbeddedH2SourceGuard.java @@ -11,7 +11,6 @@ import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.SQLException; import java.time.Duration; -import java.util.concurrent.atomic.AtomicBoolean; import javax.sql.DataSource; import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; import org.springframework.beans.factory.DisposableBean; @@ -79,21 +78,47 @@ public final class EmbeddedH2SourceGuard implements MigrationSourceGuard, Dispos private static final class ConnectionSourceLease implements MigrationSourceLease { private final Connection connection; - private final AtomicBoolean closed = new AtomicBoolean(); + private boolean closed; + private boolean callbackActive; + private Thread callbackOwner; private ConnectionSourceLease(Connection connection) { this.connection = connection; } @Override - public void close() { - if (!closed.compareAndSet(false, true)) { + public synchronized void withConnection(MigrationSourceAction action) { + if (action == null) { + throw MigrationMaintenanceException.invalidRequest(); + } + if (closed) { + throw MigrationMaintenanceException.operationConflict(); + } + if (callbackActive) { + throw MigrationMaintenanceException.operationConflict(); + } + callbackActive = true; + callbackOwner = Thread.currentThread(); + try { + action.execute(connection); + } finally { + callbackOwner = null; + callbackActive = false; + } + } + + @Override + public synchronized void close() { + if (callbackActive && callbackOwner == Thread.currentThread()) { + throw MigrationMaintenanceException.operationConflict(); + } + if (closed) { return; } try { connection.close(); + closed = true; } catch (SQLException | RuntimeException exception) { - closed.set(false); throw MigrationMaintenanceException.resumeFailure(); } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationMaintenanceException.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationMaintenanceException.java index fb21a4efc6..2b28205060 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationMaintenanceException.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationMaintenanceException.java @@ -40,7 +40,7 @@ public final class MigrationMaintenanceException extends RuntimeException { "Multi-node metadata migration is unsupported"); } - static MigrationMaintenanceException operationConflict() { + public static MigrationMaintenanceException operationConflict() { return failure(MigrationMaintenanceErrorCode.MIGRATION_OPERATION_CONFLICT, "Migration maintenance operation is already active"); } @@ -55,7 +55,7 @@ public final class MigrationMaintenanceException extends RuntimeException { "Migration maintenance acquisition was interrupted"); } - static MigrationMaintenanceException maintenanceFailure() { + public static MigrationMaintenanceException maintenanceFailure() { return failure(MigrationMaintenanceErrorCode.MIGRATION_MAINTENANCE_FAILURE, "Migration maintenance acquisition failed"); } @@ -65,7 +65,7 @@ public final class MigrationMaintenanceException extends RuntimeException { "Migration maintenance release failed"); } - static MigrationMaintenanceException invalidRequest() { + public static MigrationMaintenanceException invalidRequest() { return failure(MigrationMaintenanceErrorCode.INVALID_REQUEST, "Migration maintenance request is invalid"); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationMaintenanceLease.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationMaintenanceLease.java index 841a9544cd..dd33d11c2b 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationMaintenanceLease.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationMaintenanceLease.java @@ -10,6 +10,9 @@ package org.apache.hertzbeat.manager.maintenance; /** Owner capability for one fully acquired migration maintenance window. */ public interface MigrationMaintenanceLease extends AutoCloseable { + /** Runs synchronous work against the exact source fenced by this maintenance window. */ + void withSourceConnection(MigrationSourceAction action); + @Override void close(); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationSourceAction.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationSourceAction.java new file mode 100644 index 0000000000..5562c8a287 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationSourceAction.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.maintenance; + +import java.sql.Connection; + +/** Synchronous work scoped to the exact metadata source held by a maintenance lease. */ +@FunctionalInterface +public interface MigrationSourceAction { + + /** + * Uses the guarded source only for this callback. The action must not retain, replace, or + * independently close the connection. Only the bounded JDBC migration executor may invalidate + * it on a fail-closed timeout or unknown-outcome path; final ownership remains with the lease. + */ + void execute(Connection source); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationSourceLease.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationSourceLease.java index 5aefff2025..f0eab5f172 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationSourceLease.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/maintenance/MigrationSourceLease.java @@ -10,6 +10,9 @@ package org.apache.hertzbeat.manager.maintenance; /** Capability that releases one safe metadata-source fence. */ public interface MigrationSourceLease extends AutoCloseable { + /** Runs synchronous work against the exact source owned by this lease. */ + void withConnection(MigrationSourceAction action); + @Override void close(); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationExecutor.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationExecutor.java index ae4f11865a..6cd96ee9f2 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationExecutor.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcMetadataMigrationExecutor.java @@ -67,11 +67,22 @@ public final class JdbcMetadataMigrationExecutor implements AutoCloseable { MetadataDatabaseKind targetKind, Duration timeout, MetadataMigrationProgressSink progress) { + JdbcMetadataMigrationDeadline deadline = JdbcMetadataMigrationDeadline.start(timeout, ticker); + execute(source, target, targetKind, deadline, progress); + } + + /** Executes with an exact caller-owned monotonic budget. */ + void execute( + Connection source, + Connection target, + MetadataDatabaseKind targetKind, + JdbcMetadataMigrationDeadline deadline, + MetadataMigrationProgressSink progress) { Objects.requireNonNull(source, "source"); Objects.requireNonNull(target, "target"); Objects.requireNonNull(targetKind, "targetKind"); + Objects.requireNonNull(deadline, "deadline"); Objects.requireNonNull(progress, "progress"); - JdbcMetadataMigrationDeadline deadline = JdbcMetadataMigrationDeadline.start(timeout, ticker); JdbcMigrationConnectionScope connections = new JdbcMigrationConnectionScope( source, target, deadline, networkExecutor, abortWorker); JdbcMetadataMigrationAttempt attempt = new JdbcMetadataMigrationAttempt( diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataCopyExecutionCoordinator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataCopyExecutionCoordinator.java new file mode 100644 index 0000000000..81e5e776f2 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataCopyExecutionCoordinator.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.time.Duration; +import java.util.Objects; +import java.util.function.LongSupplier; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceException; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceLease; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceOrchestrator; +import org.apache.hertzbeat.manager.setup.api.OperationIdValidator; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** + * Runs one bounded metadata copy inside one exact maintenance lease. + * + *

A failed release retains only the completed result and exact lease. The same operation must + * call {@link #retryRelease(String)}; copy is never repeated by that recovery path. + */ +public final class MetadataCopyExecutionCoordinator { + + private final Object stateLock = new Object(); + private final MigrationMaintenanceOrchestrator maintenance; + private final JdbcMetadataMigrationExecutor executor; + private final LongSupplier ticker; + private ActiveExecution active; + + public MetadataCopyExecutionCoordinator( + MigrationMaintenanceOrchestrator maintenance, JdbcMetadataMigrationExecutor executor) { + this(maintenance, executor, System::nanoTime); + } + + MetadataCopyExecutionCoordinator( + MigrationMaintenanceOrchestrator maintenance, + JdbcMetadataMigrationExecutor executor, + LongSupplier ticker) { + this.maintenance = Objects.requireNonNull(maintenance, "maintenance"); + this.executor = Objects.requireNonNull(executor, "executor"); + this.ticker = Objects.requireNonNull(ticker, "ticker"); + } + + /** Executes one copy using the source owned by the acquired maintenance lease. */ + public void execute( + String operationId, + Connection target, + MetadataDatabaseKind targetKind, + Duration timeout, + MetadataMigrationProgressSink progress) { + requireRequest(operationId, target, targetKind, timeout, progress); + JdbcMetadataMigrationDeadline deadline = JdbcMetadataMigrationDeadline.start(timeout, ticker); + ActiveExecution execution = reserve(operationId); + try { + MigrationMaintenanceLease lease = maintenance.acquire(operationId, deadline.remainingDuration()); + if (lease == null) { + throw MigrationMaintenanceException.maintenanceFailure(); + } + execution.lease = lease; + } catch (RuntimeException | Error failure) { + clear(execution); + throw failure; + } + + MetadataCopyOutcome outcome = copy(execution.lease, target, targetKind, deadline, progress); + completeCopy(execution, outcome); + releaseAndReplay(execution); + } + + /** Retries only release of the exact pending lease and then replays the completed copy result. */ + public void retryRelease(String operationId) { + if (!OperationIdValidator.isSafe(operationId)) { + throw MigrationMaintenanceException.invalidRequest(); + } + ActiveExecution execution; + synchronized (stateLock) { + if (active == null + || active.outcome == null + || active.releasing + || !active.operationId.equals(operationId)) { + throw MigrationMaintenanceException.operationConflict(); + } + execution = active; + execution.releasing = true; + } + releaseAndReplay(execution); + } + + private MetadataCopyOutcome copy( + MigrationMaintenanceLease lease, + Connection target, + MetadataDatabaseKind targetKind, + JdbcMetadataMigrationDeadline deadline, + MetadataMigrationProgressSink progress) { + try { + deadline.remainingDuration(); + lease.withSourceConnection(source -> executor.execute( + source, target, targetKind, deadline, progress)); + return MetadataCopyOutcome.success(); + } catch (MetadataMigrationException failure) { + return MetadataCopyOutcome.stableFailure(failure.code()); + } catch (MigrationMaintenanceException failure) { + return MetadataCopyOutcome.stableMaintenanceFailure(failure); + } catch (Error fatal) { + return MetadataCopyOutcome.fatal(fatal); + } catch (RuntimeException unexpected) { + return MetadataCopyOutcome.stableFailure(MetadataMigrationErrorCode.COPY); + } + } + + private void releaseAndReplay(ActiveExecution execution) { + boolean interrupted = Thread.interrupted(); + try { + execution.lease.close(); + } catch (RuntimeException releaseFailure) { + releasePending(execution); + execution.outcome.releaseRequired(); + } catch (Error releaseFatal) { + releasePending(execution); + execution.outcome.releaseFatal(releaseFatal); + } finally { + interrupted |= Thread.interrupted(); + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + clear(execution); + execution.outcome.replay(); + } + + private void completeCopy(ActiveExecution execution, MetadataCopyOutcome outcome) { + synchronized (stateLock) { + execution.outcome = outcome; + execution.releasing = true; + } + } + + private void releasePending(ActiveExecution execution) { + synchronized (stateLock) { + if (active == execution) { + execution.releasing = false; + } + } + } + + private ActiveExecution reserve(String operationId) { + synchronized (stateLock) { + if (active != null) { + throw MigrationMaintenanceException.operationConflict(); + } + active = new ActiveExecution(operationId); + return active; + } + } + + private void clear(ActiveExecution execution) { + synchronized (stateLock) { + if (active == execution) { + active = null; + } + } + } + + private static void requireRequest( + String operationId, + Connection target, + MetadataDatabaseKind targetKind, + Duration timeout, + MetadataMigrationProgressSink progress) { + if (!OperationIdValidator.isSafe(operationId) + || target == null + || targetKind == null + || timeout == null + || progress == null) { + throw MigrationMaintenanceException.invalidRequest(); + } + } + + private static final class ActiveExecution { + + private final String operationId; + private MigrationMaintenanceLease lease; + private MetadataCopyOutcome outcome; + private boolean releasing; + + private ActiveExecution(String operationId) { + this.operationId = operationId; + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataCopyOutcome.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataCopyOutcome.java new file mode 100644 index 0000000000..19ed33a5d4 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataCopyOutcome.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceErrorCode; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceException; + +/** Completed JDBC copy result retained while its exact maintenance lease is pending release. */ +final class MetadataCopyOutcome { + + private final MetadataMigrationErrorCode stableFailure; + private final MigrationMaintenanceException stableMaintenanceFailure; + private final Error fatal; + + private MetadataCopyOutcome( + MetadataMigrationErrorCode stableFailure, + MigrationMaintenanceException stableMaintenanceFailure, + Error fatal) { + this.stableFailure = stableFailure; + this.stableMaintenanceFailure = stableMaintenanceFailure; + this.fatal = fatal; + } + + static MetadataCopyOutcome success() { + return new MetadataCopyOutcome(null, null, null); + } + + static MetadataCopyOutcome stableFailure(MetadataMigrationErrorCode code) { + return new MetadataCopyOutcome(code, null, null); + } + + static MetadataCopyOutcome stableMaintenanceFailure(MigrationMaintenanceException failure) { + return new MetadataCopyOutcome(null, failure, null); + } + + static MetadataCopyOutcome fatal(Error fatal) { + return new MetadataCopyOutcome(null, null, fatal); + } + + MetadataMigrationErrorCode stableFailure() { + return stableFailure; + } + + MigrationMaintenanceErrorCode stableMaintenanceFailure() { + return stableMaintenanceFailure == null ? null : stableMaintenanceFailure.code(); + } + + void replay() { + if (fatal != null) { + throw fatal; + } + if (stableFailure != null) { + throw new MetadataMigrationException(stableFailure); + } + if (stableMaintenanceFailure != null) { + throw stableMaintenanceFailure; + } + } + + void releaseRequired() { + if (fatal != null) { + MetadataCopyReleaseRequiredException.attachMarker(fatal, null, null); + throw fatal; + } + throw new MetadataCopyReleaseRequiredException( + stableFailure, stableMaintenanceFailure()); + } + + void releaseFatal(Error releaseFatal) { + if (fatal != null) { + releaseRequired(); + } + MetadataCopyReleaseRequiredException.attachMarker( + releaseFatal, stableFailure, stableMaintenanceFailure()); + throw releaseFatal; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataCopyReleaseRequiredException.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataCopyReleaseRequiredException.java new file mode 100644 index 0000000000..a93c374ec4 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataCopyReleaseRequiredException.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Optional; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceErrorCode; + +/** Secret-free signal that a completed copy still owns an unreleased maintenance lease. */ +public final class MetadataCopyReleaseRequiredException extends RuntimeException { + + private final MetadataMigrationErrorCode stableCopyFailure; + private final MigrationMaintenanceErrorCode stableMaintenanceFailure; + + MetadataCopyReleaseRequiredException( + MetadataMigrationErrorCode stableCopyFailure, + MigrationMaintenanceErrorCode stableMaintenanceFailure) { + super("Metadata copy maintenance release requires recovery"); + this.stableCopyFailure = stableCopyFailure; + this.stableMaintenanceFailure = stableMaintenanceFailure; + } + + public Optional stableCopyFailure() { + return Optional.ofNullable(stableCopyFailure); + } + + public Optional stableMaintenanceFailure() { + return Optional.ofNullable(stableMaintenanceFailure); + } + + static void attachMarker( + Error fatal, + MetadataMigrationErrorCode stableCopyFailure, + MigrationMaintenanceErrorCode stableMaintenanceFailure) { + for (Throwable suppressed : fatal.getSuppressed()) { + if (suppressed instanceof MetadataCopyReleaseRequiredException) { + return; + } + } + fatal.addSuppressed(new MetadataCopyReleaseRequiredException( + stableCopyFailure, stableMaintenanceFailure)); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/DefaultMigrationMaintenanceOrchestratorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/DefaultMigrationMaintenanceOrchestratorTest.java index ae0112e82a..6014553014 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/DefaultMigrationMaintenanceOrchestratorTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/DefaultMigrationMaintenanceOrchestratorTest.java @@ -16,6 +16,7 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.sql.Connection; import java.time.Duration; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -24,9 +25,11 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionCoordinator; import org.apache.hertzbeat.common.transaction.MetadataWriteMaintenanceLease; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.mockito.InOrder; import org.mockito.Mockito; +@Timeout(15) class DefaultMigrationMaintenanceOrchestratorTest { @Test @@ -72,6 +75,79 @@ class DefaultMigrationMaintenanceOrchestratorTest { order.verify(harness.authorityLease).close(); } + @Test + void compositeScopesTheExactSourceAndCannotCloseAcrossTheCallback() throws Exception { + Harness harness = harness(); + Connection source = Mockito.mock(Connection.class); + CountDownLatch callbackEntered = new CountDownLatch(1); + CountDownLatch releaseCallback = new CountDownLatch(1); + CountDownLatch closeReturned = new CountDownLatch(1); + Mockito.doAnswer(invocation -> { + MigrationSourceAction action = invocation.getArgument(0); + action.execute(source); + return null; + }).when(harness.sourceLease).withConnection(any()); + when(harness.sourceGuard.fence(any(), any())).thenReturn(harness.sourceLease); + when(harness.producerCoordinator.quiesce(any(), any())).thenReturn(harness.producerLease); + when(harness.writeCoordinator.acquire(any(), any())).thenReturn(harness.writeLease); + MigrationMaintenanceLease lease = harness.orchestrator() + .acquire("operation-a", Duration.ofSeconds(1)); + AtomicReference observed = new AtomicReference<>(); + Thread callback = Thread.ofPlatform().start(() -> lease.withSourceConnection(connection -> { + observed.set(connection); + callbackEntered.countDown(); + awaitIgnoringInterrupt(releaseCallback); + })); + Thread closer = null; + + try { + assertThat(callbackEntered.await(5, TimeUnit.SECONDS)).isTrue(); + closer = Thread.ofPlatform().start(() -> { + lease.close(); + closeReturned.countDown(); + }); + assertThat(closeReturned.await(1, TimeUnit.SECONDS)).isFalse(); + verify(harness.writeLease, never()).close(); + } finally { + releaseCallback.countDown(); + } + callback.join(5_000); + if (closer != null) { + closer.join(5_000); + } + assertThat(observed.get()).isSameAs(source); + assertThat(closeReturned.getCount()).isZero(); + verify(harness.sourceLease).withConnection(any()); + verify(harness.writeLease).close(); + } + + @Test + void compositeCloseFromItsOwnSourceCallbackFailsFast() { + Harness harness = harness(); + Connection source = Mockito.mock(Connection.class); + Mockito.doAnswer(invocation -> { + MigrationSourceAction action = invocation.getArgument(0); + action.execute(source); + return null; + }).when(harness.sourceLease).withConnection(any()); + when(harness.sourceGuard.fence(any(), any())).thenReturn(harness.sourceLease); + when(harness.producerCoordinator.quiesce(any(), any())).thenReturn(harness.producerLease); + when(harness.writeCoordinator.acquire(any(), any())).thenReturn(harness.writeLease); + MigrationMaintenanceLease lease = harness.orchestrator() + .acquire("operation-a", Duration.ofSeconds(1)); + + try { + lease.withSourceConnection(ignored -> assertThatThrownBy(lease::close) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, failure -> + assertThat(failure.code()) + .isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_OPERATION_CONFLICT))); + verify(harness.writeLease, never()).close(); + } finally { + lease.close(); + } + verify(harness.writeLease).close(); + } + @Test void deploymentAuthorityUnknownOrMultiFailsBeforeSourceOrLocalPause() { Harness unknown = harness(); @@ -319,6 +395,20 @@ class DefaultMigrationMaintenanceOrchestratorTest { return harness; } + private static void awaitIgnoringInterrupt(CountDownLatch latch) { + boolean interrupted = false; + while (latch.getCount() > 0) { + try { + latch.await(); + } catch (InterruptedException ignored) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + private record Harness( DeploymentSingletonAuthority deploymentAuthority, DeploymentSingletonLease authorityLease, diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/EmbeddedH2SourceGuardTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/EmbeddedH2SourceGuardTest.java index e84fde6a57..1b8f310c5e 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/EmbeddedH2SourceGuardTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/maintenance/EmbeddedH2SourceGuardTest.java @@ -25,11 +25,13 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import javax.sql.DataSource; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.io.TempDir; import org.mockito.Mockito; import org.springframework.boot.jdbc.autoconfigure.DataSourceProperties; import org.springframework.jdbc.datasource.DriverManagerDataSource; +@Timeout(15) class EmbeddedH2SourceGuardTest { @Test @@ -52,6 +54,93 @@ class EmbeddedH2SourceGuardTest { guard.destroy(); } + @Test + void scopesTheExactGuardedConnectionAndRejectsUseAfterClose() throws Exception { + DataSource dataSource = Mockito.mock(DataSource.class); + Connection connection = Mockito.mock(Connection.class); + DatabaseMetaData metadata = Mockito.mock(DatabaseMetaData.class); + when(dataSource.getConnection()).thenReturn(connection); + when(connection.getMetaData()).thenReturn(metadata); + when(metadata.getDatabaseProductName()).thenReturn("H2"); + when(metadata.getURL()).thenReturn("jdbc:h2:mem:manager"); + EmbeddedH2SourceGuard guard = guard(dataSource, "jdbc:h2:mem:manager"); + MigrationSourceLease lease = guard.fence("operation-a", Duration.ofSeconds(1)); + AtomicReference observed = new AtomicReference<>(); + + lease.withConnection(observed::set); + assertThat(observed.get()).isSameAs(connection); + lease.close(); + assertThatThrownBy(() -> lease.withConnection(ignored -> { })) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, failure -> + assertThat(failure.code()) + .isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_OPERATION_CONFLICT)); + guard.destroy(); + } + + @Test + void sourceCloseCannotOverlapAnActiveScopedCallback() throws Exception { + DataSource dataSource = Mockito.mock(DataSource.class); + Connection connection = Mockito.mock(Connection.class); + DatabaseMetaData metadata = Mockito.mock(DatabaseMetaData.class); + when(dataSource.getConnection()).thenReturn(connection); + when(connection.getMetaData()).thenReturn(metadata); + when(metadata.getDatabaseProductName()).thenReturn("H2"); + when(metadata.getURL()).thenReturn("jdbc:h2:mem:manager"); + EmbeddedH2SourceGuard guard = guard(dataSource, "jdbc:h2:mem:manager"); + MigrationSourceLease lease = guard.fence("operation-a", Duration.ofSeconds(1)); + CountDownLatch callbackEntered = new CountDownLatch(1); + CountDownLatch releaseCallback = new CountDownLatch(1); + CountDownLatch closeReturned = new CountDownLatch(1); + Thread callback = Thread.ofPlatform().start(() -> lease.withConnection(ignored -> { + callbackEntered.countDown(); + awaitIgnoringInterrupt(releaseCallback); + })); + Thread closer = null; + + try { + assertThat(callbackEntered.await(5, TimeUnit.SECONDS)).isTrue(); + closer = Thread.ofPlatform().start(() -> { + lease.close(); + closeReturned.countDown(); + }); + assertThat(closeReturned.await(1, TimeUnit.SECONDS)).isFalse(); + } finally { + releaseCallback.countDown(); + } + callback.join(5_000); + if (closer != null) { + closer.join(5_000); + } + assertThat(closeReturned.getCount()).isZero(); + verify(connection).close(); + guard.destroy(); + } + + @Test + void sourceCloseFromItsOwnScopedCallbackFailsFast() throws Exception { + DataSource dataSource = Mockito.mock(DataSource.class); + Connection connection = Mockito.mock(Connection.class); + DatabaseMetaData metadata = Mockito.mock(DatabaseMetaData.class); + when(dataSource.getConnection()).thenReturn(connection); + when(connection.getMetaData()).thenReturn(metadata); + when(metadata.getDatabaseProductName()).thenReturn("H2"); + when(metadata.getURL()).thenReturn("jdbc:h2:mem:manager"); + EmbeddedH2SourceGuard guard = guard(dataSource, "jdbc:h2:mem:manager"); + MigrationSourceLease lease = guard.fence("operation-a", Duration.ofSeconds(1)); + + try { + lease.withConnection(ignored -> assertThatThrownBy(lease::close) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, failure -> + assertThat(failure.code()) + .isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_OPERATION_CONFLICT))); + verify(connection, never()).close(); + } finally { + lease.close(); + guard.destroy(); + } + verify(connection).close(); + } + @Test void rejectsAmbiguousSourceAndClosesConnectionWithoutDetails() throws Exception { DataSource dataSource = Mockito.mock(DataSource.class); @@ -229,4 +318,18 @@ class EmbeddedH2SourceGuardTest { properties.setUrl(configuredUrl); return new EmbeddedH2SourceGuard(dataSource, properties); } + + private static void awaitIgnoringInterrupt(CountDownLatch latch) { + boolean interrupted = false; + while (latch.getCount() > 0) { + try { + latch.await(); + } catch (InterruptedException ignored) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataCopyExecutionCoordinatorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataCopyExecutionCoordinatorTest.java new file mode 100644 index 0000000000..574722a4ed --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataCopyExecutionCoordinatorTest.java @@ -0,0 +1,365 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.same; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceErrorCode; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceException; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceLease; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceOrchestrator; +import org.apache.hertzbeat.manager.maintenance.MigrationSourceAction; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; + +@Timeout(15) +class MetadataCopyExecutionCoordinatorTest { + + private static final Duration TIMEOUT = Duration.ofNanos(100); + + @Test + void usesExactGuardedSourceAndOneRootDeadlineBeforeReverseRelease() { + Connection source = mock(Connection.class); + Connection target = mock(Connection.class); + MigrationMaintenanceOrchestrator maintenance = mock(MigrationMaintenanceOrchestrator.class); + MigrationMaintenanceLease lease = sourceLease(source); + JdbcMetadataMigrationExecutor executor = mock(JdbcMetadataMigrationExecutor.class); + AtomicLong ticker = new AtomicLong(10); + when(maintenance.acquire(eq("operation-a"), any())).thenAnswer(invocation -> { + assertThat((Duration) invocation.getArgument(1)).isEqualTo(TIMEOUT); + ticker.addAndGet(30); + return lease; + }); + MetadataCopyExecutionCoordinator coordinator = + new MetadataCopyExecutionCoordinator(maintenance, executor, ticker::get); + + coordinator.execute("operation-a", target, MetadataDatabaseKind.MYSQL, TIMEOUT, + MetadataMigrationProgressSink.NO_OP); + + ArgumentCaptor deadline = + ArgumentCaptor.forClass(JdbcMetadataMigrationDeadline.class); + InOrder order = inOrder(maintenance, lease, executor); + order.verify(maintenance).acquire(eq("operation-a"), any()); + order.verify(lease).withSourceConnection(any()); + order.verify(executor).execute(same(source), same(target), eq(MetadataDatabaseKind.MYSQL), + deadline.capture(), same(MetadataMigrationProgressSink.NO_OP)); + order.verify(lease).close(); + assertThat(deadline.getValue().remainingNanos()).isEqualTo(70); + } + + @Test + void releasesBeforeReplayingStableCopyFailure() { + Connection source = mock(Connection.class); + Connection target = mock(Connection.class); + MigrationMaintenanceOrchestrator maintenance = mock(MigrationMaintenanceOrchestrator.class); + MigrationMaintenanceLease lease = sourceLease(source); + JdbcMetadataMigrationExecutor executor = mock(JdbcMetadataMigrationExecutor.class); + when(maintenance.acquire(any(), any())).thenReturn(lease); + doThrow(new MetadataMigrationException(MetadataMigrationErrorCode.VERIFICATION)) + .when(executor).execute(same(source), same(target), any(), anyDeadline(), any()); + MetadataCopyExecutionCoordinator coordinator = coordinator(maintenance, executor); + + assertThatThrownBy(() -> coordinator.execute("operation-a", target, + MetadataDatabaseKind.POSTGRESQL, Duration.ofSeconds(1), + MetadataMigrationProgressSink.NO_OP)) + .isInstanceOfSatisfying(MetadataMigrationException.class, failure -> + assertThat(failure.code()).isEqualTo(MetadataMigrationErrorCode.VERIFICATION)); + + InOrder order = inOrder(executor, lease); + order.verify(executor).execute(any(), any(), any(), anyDeadline(), any()); + order.verify(lease).close(); + } + + @Test + void releasesBeforeReplayingFatalCopyError() { + Connection source = mock(Connection.class); + Connection target = mock(Connection.class); + MigrationMaintenanceOrchestrator maintenance = mock(MigrationMaintenanceOrchestrator.class); + MigrationMaintenanceLease lease = sourceLease(source); + JdbcMetadataMigrationExecutor executor = mock(JdbcMetadataMigrationExecutor.class); + AssertionError fatal = new AssertionError("fatal copy"); + when(maintenance.acquire(any(), any())).thenReturn(lease); + doThrow(fatal).when(executor).execute(any(), any(), any(), anyDeadline(), any()); + MetadataCopyExecutionCoordinator coordinator = coordinator(maintenance, executor); + + assertThatThrownBy(() -> coordinator.execute("operation-a", target, + MetadataDatabaseKind.MYSQL, Duration.ofSeconds(1), + MetadataMigrationProgressSink.NO_OP)) + .isSameAs(fatal); + + InOrder order = inOrder(executor, lease); + order.verify(executor).execute(any(), any(), any(), anyDeadline(), any()); + order.verify(lease).close(); + } + + @Test + void releasesBeforeReplayingStableMaintenanceFailureWithoutMappingItToCopy() { + Connection target = mock(Connection.class); + MigrationMaintenanceOrchestrator maintenance = mock(MigrationMaintenanceOrchestrator.class); + MigrationMaintenanceLease lease = mock(MigrationMaintenanceLease.class); + JdbcMetadataMigrationExecutor executor = mock(JdbcMetadataMigrationExecutor.class); + when(maintenance.acquire(any(), any())).thenReturn(lease); + doThrow(MigrationMaintenanceException.sourceUnavailable()) + .when(lease).withSourceConnection(any()); + MetadataCopyExecutionCoordinator coordinator = coordinator(maintenance, executor); + + assertThatThrownBy(() -> coordinator.execute("operation-a", target, + MetadataDatabaseKind.MYSQL, Duration.ofSeconds(1), + MetadataMigrationProgressSink.NO_OP)) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, failure -> + assertThat(failure.code()) + .isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_SOURCE_UNAVAILABLE)); + + InOrder order = inOrder(lease, executor); + order.verify(lease).withSourceConnection(any()); + order.verify(lease).close(); + verifyNoInteractions(executor); + } + + @Test + void expiredAcquisitionReleasesLeaseWithoutEnteringSourceScopeOrCopy() { + Connection target = mock(Connection.class); + MigrationMaintenanceOrchestrator maintenance = mock(MigrationMaintenanceOrchestrator.class); + MigrationMaintenanceLease lease = mock(MigrationMaintenanceLease.class); + JdbcMetadataMigrationExecutor executor = mock(JdbcMetadataMigrationExecutor.class); + AtomicLong ticker = new AtomicLong(10); + when(maintenance.acquire(any(), any())).thenAnswer(invocation -> { + ticker.addAndGet(101); + return lease; + }); + MetadataCopyExecutionCoordinator coordinator = + new MetadataCopyExecutionCoordinator(maintenance, executor, ticker::get); + + assertThatThrownBy(() -> coordinator.execute("operation-a", target, + MetadataDatabaseKind.MYSQL, TIMEOUT, MetadataMigrationProgressSink.NO_OP)) + .isInstanceOfSatisfying(MetadataMigrationException.class, failure -> + assertThat(failure.code()).isEqualTo(MetadataMigrationErrorCode.TIMEOUT)); + + verify(lease, never()).withSourceConnection(any()); + verifyNoInteractions(executor); + verify(lease).close(); + } + + @Test + void pendingReleaseRetainsStableOutcomeAndRetriesOnlyTheExactLease() { + Connection source = mock(Connection.class); + Connection target = mock(Connection.class); + Connection foreignTarget = mock(Connection.class); + MigrationMaintenanceOrchestrator maintenance = mock(MigrationMaintenanceOrchestrator.class); + MigrationMaintenanceLease lease = sourceLease(source); + JdbcMetadataMigrationExecutor executor = mock(JdbcMetadataMigrationExecutor.class); + when(maintenance.acquire(any(), any())).thenReturn(lease); + doThrow(new MetadataMigrationException(MetadataMigrationErrorCode.COPY)) + .when(executor).execute(any(), any(), any(), anyDeadline(), any()); + doThrow(new IllegalStateException("private release")) + .doNothing().when(lease).close(); + MetadataCopyExecutionCoordinator coordinator = coordinator(maintenance, executor); + + assertThatThrownBy(() -> coordinator.execute("operation-a", target, + MetadataDatabaseKind.MYSQL, Duration.ofSeconds(1), + MetadataMigrationProgressSink.NO_OP)) + .isInstanceOfSatisfying(MetadataCopyReleaseRequiredException.class, failure -> { + assertThat(failure.stableCopyFailure()).contains(MetadataMigrationErrorCode.COPY); + assertThat(failure).hasNoCause(); + assertThat(failure.getMessage()).doesNotContain("private"); + }); + assertConflict(() -> coordinator.execute("operation-b", foreignTarget, + MetadataDatabaseKind.POSTGRESQL, Duration.ofSeconds(1), MetadataMigrationProgressSink.NO_OP)); + assertConflict(() -> coordinator.retryRelease("operation-b")); + + assertThatThrownBy(() -> coordinator.retryRelease("operation-a")) + .isInstanceOfSatisfying(MetadataMigrationException.class, failure -> + assertThat(failure.code()).isEqualTo(MetadataMigrationErrorCode.COPY)); + verify(maintenance).acquire(any(), any()); + verify(executor).execute(any(), any(), any(), anyDeadline(), any()); + verify(lease, times(2)).close(); + verifyNoInteractions(foreignTarget); + } + + @Test + void pendingReleaseCarriesOnlyTheStableMaintenanceCodeAndRetriesWithoutCopy() { + Connection target = mock(Connection.class); + MigrationMaintenanceOrchestrator maintenance = mock(MigrationMaintenanceOrchestrator.class); + MigrationMaintenanceLease lease = mock(MigrationMaintenanceLease.class); + JdbcMetadataMigrationExecutor executor = mock(JdbcMetadataMigrationExecutor.class); + when(maintenance.acquire(any(), any())).thenReturn(lease); + doThrow(MigrationMaintenanceException.sourceUnavailable()) + .when(lease).withSourceConnection(any()); + doThrow(new IllegalStateException("private release")) + .doNothing().when(lease).close(); + MetadataCopyExecutionCoordinator coordinator = coordinator(maintenance, executor); + + assertThatThrownBy(() -> coordinator.execute("operation-a", target, + MetadataDatabaseKind.MYSQL, Duration.ofSeconds(1), + MetadataMigrationProgressSink.NO_OP)) + .isInstanceOfSatisfying(MetadataCopyReleaseRequiredException.class, failure -> { + assertThat(failure.stableCopyFailure()).isEmpty(); + assertThat(failure.stableMaintenanceFailure()) + .contains(MigrationMaintenanceErrorCode.MIGRATION_SOURCE_UNAVAILABLE); + assertThat(failure).hasNoCause(); + assertThat(failure.getMessage()).doesNotContain("private"); + }); + assertThatThrownBy(() -> coordinator.retryRelease("operation-a")) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, failure -> + assertThat(failure.code()) + .isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_SOURCE_UNAVAILABLE)); + verifyNoInteractions(executor); + verify(lease, times(2)).close(); + } + + @Test + void fatalCopyRemainsPrimaryWhenRuntimeReleaseNeedsRetry() { + Connection source = mock(Connection.class); + Connection target = mock(Connection.class); + MigrationMaintenanceOrchestrator maintenance = mock(MigrationMaintenanceOrchestrator.class); + MigrationMaintenanceLease lease = sourceLease(source); + JdbcMetadataMigrationExecutor executor = mock(JdbcMetadataMigrationExecutor.class); + AssertionError fatal = new AssertionError("fatal copy"); + when(maintenance.acquire(any(), any())).thenReturn(lease); + doThrow(fatal).when(executor).execute(any(), any(), any(), anyDeadline(), any()); + doThrow(new IllegalStateException("private release")) + .doNothing().when(lease).close(); + MetadataCopyExecutionCoordinator coordinator = coordinator(maintenance, executor); + + assertThatThrownBy(() -> coordinator.execute("operation-a", target, + MetadataDatabaseKind.MYSQL, Duration.ofSeconds(1), + MetadataMigrationProgressSink.NO_OP)) + .isSameAs(fatal); + assertThat(fatal.getSuppressed()).singleElement() + .isInstanceOf(MetadataCopyReleaseRequiredException.class); + assertThatThrownBy(() -> coordinator.retryRelease("operation-a")).isSameAs(fatal); + verify(executor).execute(any(), any(), any(), anyDeadline(), any()); + verify(lease, times(2)).close(); + } + + @Test + void fatalCopyRemainsPrimaryWhenReleaseAlsoThrowsError() { + Connection source = mock(Connection.class); + Connection target = mock(Connection.class); + MigrationMaintenanceOrchestrator maintenance = mock(MigrationMaintenanceOrchestrator.class); + MigrationMaintenanceLease lease = sourceLease(source); + JdbcMetadataMigrationExecutor executor = mock(JdbcMetadataMigrationExecutor.class); + AssertionError copyFatal = new AssertionError("fatal copy"); + AssertionError releaseFatal = new AssertionError("fatal release"); + when(maintenance.acquire(any(), any())).thenReturn(lease); + doThrow(copyFatal).when(executor).execute(any(), any(), any(), anyDeadline(), any()); + doThrow(releaseFatal).doNothing().when(lease).close(); + MetadataCopyExecutionCoordinator coordinator = coordinator(maintenance, executor); + + assertThatThrownBy(() -> coordinator.execute("operation-a", target, + MetadataDatabaseKind.MYSQL, Duration.ofSeconds(1), + MetadataMigrationProgressSink.NO_OP)) + .isSameAs(copyFatal); + assertThat(copyFatal.getSuppressed()).singleElement() + .isInstanceOf(MetadataCopyReleaseRequiredException.class); + assertThat(copyFatal.getSuppressed()).doesNotContain(releaseFatal); + assertThatThrownBy(() -> coordinator.retryRelease("operation-a")).isSameAs(copyFatal); + verify(executor).execute(any(), any(), any(), anyDeadline(), any()); + verify(lease, times(2)).close(); + } + + @Test + void releaseErrorRemainsPrimaryAndExactLeaseCanStillConverge() { + Connection source = mock(Connection.class); + Connection target = mock(Connection.class); + MigrationMaintenanceOrchestrator maintenance = mock(MigrationMaintenanceOrchestrator.class); + MigrationMaintenanceLease lease = sourceLease(source); + JdbcMetadataMigrationExecutor executor = mock(JdbcMetadataMigrationExecutor.class); + AssertionError releaseFatal = new AssertionError("fatal release"); + when(maintenance.acquire(any(), any())).thenReturn(lease); + doThrow(releaseFatal).doNothing().when(lease).close(); + MetadataCopyExecutionCoordinator coordinator = coordinator(maintenance, executor); + + assertThatThrownBy(() -> coordinator.execute("operation-a", target, + MetadataDatabaseKind.MYSQL, Duration.ofSeconds(1), + MetadataMigrationProgressSink.NO_OP)) + .isSameAs(releaseFatal); + assertThat(releaseFatal.getSuppressed()).singleElement() + .isInstanceOf(MetadataCopyReleaseRequiredException.class); + coordinator.retryRelease("operation-a"); + verify(executor).execute(any(), any(), any(), anyDeadline(), any()); + verify(lease, times(2)).close(); + } + + @Test + void interruptBitIsClearedOnlyDuringMandatoryReleaseAndThenRestored() { + Connection source = mock(Connection.class); + Connection target = mock(Connection.class); + MigrationMaintenanceOrchestrator maintenance = mock(MigrationMaintenanceOrchestrator.class); + MigrationMaintenanceLease lease = sourceLease(source); + JdbcMetadataMigrationExecutor executor = mock(JdbcMetadataMigrationExecutor.class); + when(maintenance.acquire(any(), any())).thenReturn(lease); + doAnswer(invocation -> { + Thread.currentThread().interrupt(); + return null; + }).when(executor).execute(any(), any(), any(), anyDeadline(), any()); + doAnswer(invocation -> { + assertThat(Thread.currentThread().isInterrupted()).isFalse(); + return null; + }).when(lease).close(); + MetadataCopyExecutionCoordinator coordinator = coordinator(maintenance, executor); + + try { + coordinator.execute("operation-a", target, MetadataDatabaseKind.MYSQL, + Duration.ofSeconds(1), MetadataMigrationProgressSink.NO_OP); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } finally { + Thread.interrupted(); + } + } + + private static MetadataCopyExecutionCoordinator coordinator( + MigrationMaintenanceOrchestrator maintenance, JdbcMetadataMigrationExecutor executor) { + return new MetadataCopyExecutionCoordinator(maintenance, executor, System::nanoTime); + } + + private static JdbcMetadataMigrationDeadline anyDeadline() { + return any(JdbcMetadataMigrationDeadline.class); + } + + private static MigrationMaintenanceLease sourceLease(Connection source) { + MigrationMaintenanceLease lease = mock(MigrationMaintenanceLease.class); + doAnswer(invocation -> { + MigrationSourceAction action = invocation.getArgument(0); + action.execute(source); + return null; + }).when(lease).withSourceConnection(any()); + return lease; + } + + private static void assertConflict(ThrowingAction action) { + assertThatThrownBy(action::run) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, failure -> + assertThat(failure.code()) + .isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_OPERATION_CONFLICT)); + } + + @FunctionalInterface + private interface ThrowingAction { + void run(); + } +} From 4322c0d21d32e7d1565f9dd555aa620927559255 Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 09:49:26 +0800 Subject: [PATCH 46/71] Bound target schema provisioning --- .../setup/workflow/FlywaySchemaHistory.java | 69 +++-- .../FlywayTargetSchemaProvisioner.java | 173 ++++------- .../FlywayTargetSchemaProvisioningCore.java | 258 ++++++++++++++++ .../FlywayTargetSchemaProvisioningWork.java | 119 ++++++++ .../workflow/JdbcTargetSchemaObjectState.java | 42 +-- .../setup/workflow/JdbcTargetSchemaState.java | 59 +++- .../TargetSchemaConnectionDisposition.java | 14 + .../setup/workflow/TargetSchemaContract.java | 36 ++- .../workflow/TargetSchemaJdbcBudget.java | 69 +++++ .../TargetSchemaProvisioningException.java | 13 + .../TargetSchemaProvisioningFailure.java | 5 +- .../TargetSchemaProvisioningOutcome.java | 18 ++ .../TargetSchemaProvisioningWork.java | 17 ++ .../workflow/TargetSchemaSqlFailure.java | 38 +++ ...lywayTargetSchemaConnectionBudgetTest.java | 156 ++++++++++ .../FlywayTargetSchemaProvisionerTest.java | 84 +++++- ...lywayTargetSchemaProvisioningCoreTest.java | 269 +++++++++++++++++ ...wayTargetSchemaProvisioningSafetyTest.java | 280 ++++++++++++++++++ ...lywayTargetSchemaProvisioningWorkTest.java | 167 +++++++++++ .../workflow/TargetSchemaJdbcBudgetTest.java | 156 ++++++++++ .../TargetSchemaJdbcMetadataBudgetTest.java | 128 ++++++++ .../TargetSchemaProvisionerDatabaseTest.java | 29 +- 22 files changed, 2020 insertions(+), 179 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioningCore.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioningWork.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaConnectionDisposition.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaJdbcBudget.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioningOutcome.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioningWork.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaSqlFailure.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaConnectionBudgetTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioningCoreTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioningSafetyTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioningWorkTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaJdbcBudgetTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaJdbcMetadataBudgetTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywaySchemaHistory.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywaySchemaHistory.java index 470a4060f8..b7f9fdd90d 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywaySchemaHistory.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywaySchemaHistory.java @@ -39,24 +39,31 @@ final class FlywaySchemaHistory { } boolean isCurrent(Connection connection, TargetSchemaBaseline baseline) throws SQLException { - return isCurrent(connection, baseline, 0); + return isCurrent(connection, baseline, TargetSchemaJdbcBudget.none()); } boolean isCurrent( Connection connection, TargetSchemaBaseline baseline, int queryTimeoutSeconds) throws SQLException { - Set currentTables = currentBaselineTables(connection); + return isCurrent(connection, baseline, TargetSchemaJdbcBudget.fixed(queryTimeoutSeconds)); + } + + boolean isCurrent( + Connection connection, + TargetSchemaBaseline baseline, + TargetSchemaJdbcBudget budget) throws SQLException { + Set currentTables = currentBaselineTables(connection, budget); if (!currentTables.contains(TABLE)) { return false; } String sql = "SELECT installed_rank, version, type, script, checksum, success FROM " + TABLE; try (Statement statement = connection.createStatement()) { - if (queryTimeoutSeconds > 0) { - statement.setQueryTimeout(queryTimeoutSeconds); - } + budget.apply(statement); try (ResultSet result = statement.executeQuery(sql)) { + budget.check(); if (!result.next()) { throw unexpectedTargetState(); } + budget.check(); boolean current = result.getInt("installed_rank") == 1 && TargetSchemaBaseline.VERSION.equals(result.getString("version")) && TargetSchemaBaseline.TYPE.equals(result.getString("type")) @@ -64,10 +71,11 @@ final class FlywaySchemaHistory { && baseline.checksum() == result.getInt("checksum") && !result.wasNull() && result.getBoolean("success"); + budget.check(); if (!current || result.next() || !currentTables.contains(TargetSchemaContract.TABLE) || !currentTables.containsAll(baseline.expectedTables()) || !new TargetSchemaContract(kind) - .matches(connection, baseline.expectedTables(), queryTimeoutSeconds)) { + .matches(connection, baseline.expectedTables(), budget)) { throw unexpectedTargetState(); } return true; @@ -76,7 +84,11 @@ final class FlywaySchemaHistory { } void requireEmptyTarget(Connection connection) throws SQLException { - if (!currentCatalogSchemaObjects(connection).isEmpty()) { + requireEmptyTarget(connection, TargetSchemaJdbcBudget.none()); + } + + void requireEmptyTarget(Connection connection, TargetSchemaJdbcBudget budget) throws SQLException { + if (!currentCatalogSchemaObjects(connection, null, budget).isEmpty()) { throw unexpectedTargetState(); } } @@ -86,9 +98,19 @@ final class FlywaySchemaHistory { TargetSchemaBaseline baseline, String installedBy, int executionTimeMillis) throws SQLException { - new TargetSchemaContract(kind).record(connection, baseline.expectedTables()); + record(connection, baseline, installedBy, executionTimeMillis, TargetSchemaJdbcBudget.none()); + } + + void record( + Connection connection, + TargetSchemaBaseline baseline, + String installedBy, + int executionTimeMillis, + TargetSchemaJdbcBudget budget) throws SQLException { + new TargetSchemaContract(kind).record(connection, baseline.expectedTables(), budget); try (Statement statement = connection.createStatement()) { for (String sql : createStatements()) { + budget.apply(statement); statement.execute(sql); } } @@ -96,6 +118,7 @@ final class FlywaySchemaHistory { + " (installed_rank, version, description, type, script, checksum, installed_by, execution_time, success)" + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"; try (PreparedStatement statement = connection.prepareStatement(insert)) { + budget.apply(statement); statement.setInt(1, 1); statement.setString(2, TargetSchemaBaseline.VERSION); statement.setString(3, TargetSchemaBaseline.DESCRIPTION); @@ -105,6 +128,7 @@ final class FlywaySchemaHistory { statement.setString(7, abbreviate(installedBy, 100)); statement.setInt(8, executionTimeMillis); statement.setBoolean(9, true); + budget.apply(statement); statement.executeUpdate(); } } @@ -128,23 +152,34 @@ final class FlywaySchemaHistory { return new String[]{table, "CREATE INDEX flyway_schema_history_s_idx ON " + TABLE + " (success)"}; } - private Set currentBaselineTables(Connection connection) throws SQLException { - return currentCatalogSchemaObjects(connection, new String[]{"TABLE"}); + private Set currentBaselineTables( + Connection connection, TargetSchemaJdbcBudget budget) throws SQLException { + return currentCatalogSchemaObjects(connection, new String[]{"TABLE"}, budget); } - private Set currentCatalogSchemaObjects(Connection connection) throws SQLException { - return currentCatalogSchemaObjects(connection, null); - } - - private Set currentCatalogSchemaObjects(Connection connection, String[] types) throws SQLException { + private Set currentCatalogSchemaObjects( + Connection connection, + String[] types, + TargetSchemaJdbcBudget budget) throws SQLException { + budget.check(); DatabaseMetaData metadata = connection.getMetaData(); - String schema = kind == MetadataDatabaseKind.POSTGRESQL ? connection.getSchema() : null; + budget.check(); + String catalog = connection.getCatalog(); + budget.check(); + String schema = null; + if (kind == MetadataDatabaseKind.POSTGRESQL) { + schema = connection.getSchema(); + budget.check(); + } Set names = new HashSet<>(); - try (ResultSet objects = metadata.getTables(connection.getCatalog(), schema, "%", types)) { + try (ResultSet objects = metadata.getTables(catalog, schema, "%", types)) { + budget.check(); while (objects.next()) { + budget.check(); names.add(objects.getString("TABLE_NAME").toLowerCase(Locale.ROOT)); } } + budget.check(); return Set.copyOf(names); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioner.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioner.java index 9d470811b9..627cca502f 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioner.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioner.java @@ -2,159 +2,90 @@ * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * The ASF licenses this file to You under the Apache License, Version 2.0. */ package org.apache.hertzbeat.manager.setup.workflow; -import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; -import java.sql.Statement; +import java.time.Duration; import java.util.Objects; -import java.util.concurrent.locks.ReentrantLock; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; -/** Applies the static baseline and writes a history row compatible with subsequent standard Flyway runs. */ +/** Compatibility adapter around caller-owned, deadline-aware target schema provisioning. */ public final class FlywayTargetSchemaProvisioner implements TargetSchemaProvisioner { - // Admission rejects multi-node migration. The lock prevents concurrent work in this JVM, while a failed MySQL DDL - // sequence can leave partial state that deliberately fails the next precondition instead of pretending to resume. - private static final ReentrantLock PROVISIONING_LOCK = new ReentrantLock(); + private static final Duration COMPATIBILITY_TIMEOUT = Duration.ofMinutes(5); @Override public void provision(MetadataDatabaseConfiguration target) { Objects.requireNonNull(target, "target"); MetadataDatabaseKind kind = supportedKind(target.kind()); - PROVISIONING_LOCK.lock(); - try { - provisionLocked(target, kind); - } finally { - PROVISIONING_LOCK.unlock(); - } - } - - private static void provisionLocked(MetadataDatabaseConfiguration target, MetadataDatabaseKind kind) { Connection connection; try { connection = DriverManager.getConnection(target.jdbcUrl(), target.username(), target.password()); - } catch (SQLException exception) { - throw failure(kind, TargetSchemaProvisioningFailure.Phase.CONNECTION, exception); + } catch (SQLException failure) { + throw failure(kind, TargetSchemaProvisioningFailure.Phase.CONNECTION, failure); } - boolean completed = false; + provisionOwned(connection, kind, () -> provision(connection, kind, + JdbcMetadataMigrationDeadline.start(COMPATIBILITY_TIMEOUT, System::nanoTime))); + } + + /** Provisions without opening, closing, or retaining the caller-owned connection. */ + TargetSchemaProvisioningOutcome provision( + Connection connection, + MetadataDatabaseKind kind, + JdbcMetadataMigrationDeadline deadline) { + MetadataDatabaseKind supported = supportedKind(kind); + return new FlywayTargetSchemaProvisioningCore(new FlywayTargetSchemaProvisioningWork(supported)) + .provision(connection, supported, deadline); + } + + void provisionOwned(Connection connection, MetadataDatabaseKind kind, Runnable action) { + Objects.requireNonNull(connection, "connection"); + Objects.requireNonNull(action, "action"); + Throwable primary = null; try { - configureTransaction(connection, kind); - provision(connection, target, kind); - commitTransaction(connection, kind); - completed = true; - } catch (TargetSchemaProvisioningException exception) { - rollbackTransaction(connection, kind); - throw exception; + action.run(); + } catch (RuntimeException failure) { + primary = failure; + throw failure; + } catch (Error failure) { + primary = failure; + throw failure; } finally { - if (!completed) { - closeQuietly(connection); - } + closeOwned(connection, supportedKind(kind), primary); } + } + + private static void closeOwned( + Connection connection, + MetadataDatabaseKind kind, + Throwable primary) { try { connection.close(); - } catch (SQLException exception) { - throw failure(kind, TargetSchemaProvisioningFailure.Phase.CLEANUP, exception); - } - } - - private static void configureTransaction(Connection connection, MetadataDatabaseKind kind) { - if (kind == MetadataDatabaseKind.POSTGRESQL) { - try { - connection.setAutoCommit(false); - } catch (SQLException exception) { - throw failure(kind, TargetSchemaProvisioningFailure.Phase.TRANSACTION, exception); + } catch (SQLException | RuntimeException closeFailure) { + if (primary == null) { + throw failure(kind, TargetSchemaProvisioningFailure.Phase.CLEANUP, closeFailure); + } + } catch (Error closeFatal) { + if (primary == null) { + throw closeFatal; } } } - private static void commitTransaction(Connection connection, MetadataDatabaseKind kind) { - if (kind == MetadataDatabaseKind.POSTGRESQL) { - try { - connection.commit(); - } catch (SQLException exception) { - throw failure(kind, TargetSchemaProvisioningFailure.Phase.TRANSACTION, exception); - } - } - } - - private static void rollbackTransaction(Connection connection, MetadataDatabaseKind kind) { - if (kind == MetadataDatabaseKind.POSTGRESQL) { - try { - connection.rollback(); - } catch (SQLException ignored) { - // Preserve the sanitized failure from the operation phase. - } - } - } - - private static void closeQuietly(Connection connection) { - try { - connection.close(); - } catch (SQLException ignored) { - // Never attach raw driver diagnostics to the sanitized operation failure. - } - } - - private static void provision( - Connection connection, MetadataDatabaseConfiguration target, MetadataDatabaseKind kind) { - TargetSchemaBaseline baseline; - try { - baseline = TargetSchemaBaseline.load(kind); - } catch (IOException exception) { - throw failure(kind, TargetSchemaProvisioningFailure.Phase.BASELINE_RESOURCE, exception); - } - FlywaySchemaHistory history = new FlywaySchemaHistory(kind); - try { - if (history.isCurrent(connection, baseline)) { - return; - } - history.requireEmptyTarget(connection); - } catch (SQLException exception) { - throw failure(kind, TargetSchemaProvisioningFailure.Phase.PRECONDITION, exception); - } - int executionTimeMillis; - try { - executionTimeMillis = execute(connection, baseline); - } catch (SQLException exception) { - throw failure(kind, TargetSchemaProvisioningFailure.Phase.BASELINE_EXECUTION, exception); - } - try { - history.record(connection, baseline, target.username(), executionTimeMillis); - } catch (SQLException exception) { - throw failure(kind, TargetSchemaProvisioningFailure.Phase.HISTORY_WRITE, exception); - } - } - - private static int execute(Connection connection, TargetSchemaBaseline baseline) throws SQLException { - long startedAt = System.nanoTime(); - try (Statement statement = connection.createStatement()) { - for (String sql : baseline.statements()) { - statement.execute(sql); - } - } - return Math.toIntExact(Math.min(Integer.MAX_VALUE, (System.nanoTime() - startedAt) / 1_000_000L)); - } - private static TargetSchemaProvisioningException failure( - MetadataDatabaseKind kind, TargetSchemaProvisioningFailure.Phase phase, Throwable exception) { - return new TargetSchemaProvisioningException(kind, TargetSchemaProvisioningFailure.from(phase, exception)); + MetadataDatabaseKind kind, + TargetSchemaProvisioningFailure.Phase phase, + Throwable cause) { + return new TargetSchemaProvisioningException( + kind, + TargetSchemaProvisioningFailure.from(phase, cause), + TargetSchemaConnectionDisposition.DISCARD_REQUIRED); } private static MetadataDatabaseKind supportedKind(MetadataDatabaseKind kind) { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioningCore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioningCore.java new file mode 100644 index 0000000000..076dec3a3a --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioningCore.java @@ -0,0 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.util.Locale; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Owns schema admission and PostgreSQL transaction state, but never the caller connection. */ +final class FlywayTargetSchemaProvisioningCore { + + private static final ReentrantLock SHARED_LOCK = new ReentrantLock(); + private final ReentrantLock lock; + private final TargetSchemaProvisioningWork work; + + FlywayTargetSchemaProvisioningCore(TargetSchemaProvisioningWork work) { + this(SHARED_LOCK, work); + } + + FlywayTargetSchemaProvisioningCore(ReentrantLock lock, TargetSchemaProvisioningWork work) { + this.lock = Objects.requireNonNull(lock, "lock"); + this.work = Objects.requireNonNull(work, "work"); + } + + TargetSchemaProvisioningOutcome provision( + Connection connection, + MetadataDatabaseKind kind, + JdbcMetadataMigrationDeadline deadline) { + Objects.requireNonNull(connection, "connection"); + MetadataDatabaseKind supported = supportedKind(kind); + Objects.requireNonNull(deadline, "deadline"); + boolean acquired; + try { + acquired = lock.tryLock(deadline.remainingNanos(), TimeUnit.NANOSECONDS); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw deadlineFailure(supported, TargetSchemaConnectionDisposition.REUSABLE); + } + if (!acquired) { + throw deadlineFailure(supported, TargetSchemaConnectionDisposition.REUSABLE); + } + try { + TargetSchemaJdbcBudget budget = new TargetSchemaJdbcBudget(deadline); + try { + budget.check(); + } catch (MetadataMigrationException timeout) { + throw deadlineFailure(supported, TargetSchemaConnectionDisposition.REUSABLE); + } + requireIdleWritableConnection(connection, supported, budget); + return supported == MetadataDatabaseKind.POSTGRESQL + ? provisionPostgresql(connection, supported, budget) + : provisionMysql(connection, supported, budget); + } finally { + lock.unlock(); + } + } + + private TargetSchemaProvisioningOutcome provisionMysql( + Connection connection, + MetadataDatabaseKind kind, + TargetSchemaJdbcBudget budget) { + try { + budget.check(); + work.provision(connection, budget); + return reusable(); + } catch (TargetSchemaProvisioningException failure) { + throw failure; + } catch (MetadataMigrationException timeout) { + throw deadlineFailure(kind, TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + } catch (RuntimeException unexpected) { + throw failure(kind, TargetSchemaProvisioningFailure.Phase.BASELINE_EXECUTION, + unexpected, TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + } + } + + private TargetSchemaProvisioningOutcome provisionPostgresql( + Connection connection, + MetadataDatabaseKind kind, + TargetSchemaJdbcBudget budget) { + try { + connection.setAutoCommit(false); + } catch (SQLException | RuntimeException failure) { + throw failure(kind, TargetSchemaProvisioningFailure.Phase.TRANSACTION, + failure, TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + } + try { + budget.check(); + work.provision(connection, budget); + } catch (TargetSchemaProvisioningException failure) { + throw rollbackKnownFailure(connection, kind, failure); + } catch (MetadataMigrationException timeout) { + throw rollbackKnownFailure(connection, kind, + deadlineFailure(kind, TargetSchemaConnectionDisposition.DISCARD_REQUIRED)); + } catch (RuntimeException unexpected) { + throw rollbackKnownFailure(connection, kind, + failure(kind, TargetSchemaProvisioningFailure.Phase.BASELINE_EXECUTION, + unexpected, TargetSchemaConnectionDisposition.DISCARD_REQUIRED)); + } catch (Error fatal) { + cleanupAfterFatal(connection); + throw fatal; + } + try { + budget.check(); + connection.commit(); + } catch (MetadataMigrationException timeout) { + throw rollbackKnownFailure(connection, kind, + deadlineFailure(kind, TargetSchemaConnectionDisposition.DISCARD_REQUIRED)); + } catch (SQLException | RuntimeException failure) { + throw failure(kind, TargetSchemaProvisioningFailure.Phase.COMMIT_OUTCOME_UNKNOWN, + failure, TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + } + return restoreAfterKnownSuccess(connection) ? reusable() : discardRequired(); + } + + private static void requireIdleWritableConnection( + Connection connection, + MetadataDatabaseKind kind, + TargetSchemaJdbcBudget budget) { + boolean autoCommit; + boolean readOnly; + try { + budget.check(); + autoCommit = connection.getAutoCommit(); + budget.check(); + readOnly = connection.isReadOnly(); + budget.check(); + } catch (MetadataMigrationException timeout) { + throw deadlineFailure(kind, TargetSchemaConnectionDisposition.REUSABLE); + } catch (SQLException | RuntimeException failure) { + throw failure(kind, TargetSchemaProvisioningFailure.Phase.TRANSACTION, + failure, TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + } + if (!autoCommit || readOnly) { + throw failure(kind, TargetSchemaProvisioningFailure.Phase.TRANSACTION, + null, TargetSchemaConnectionDisposition.REUSABLE); + } + String product; + try { + DatabaseMetaData metadata = connection.getMetaData(); + budget.check(); + product = metadata.getDatabaseProductName().toLowerCase(Locale.ROOT); + budget.check(); + } catch (MetadataMigrationException timeout) { + throw deadlineFailure(kind, TargetSchemaConnectionDisposition.REUSABLE); + } catch (SQLException | RuntimeException failure) { + throw failure(kind, TargetSchemaProvisioningFailure.Phase.PRECONDITION, + failure, TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + } + boolean matches = switch (kind) { + case MYSQL -> product.contains("mysql"); + case POSTGRESQL -> product.contains("postgresql"); + case H2 -> false; + }; + if (!matches) { + throw failure(kind, TargetSchemaProvisioningFailure.Phase.PRECONDITION, + null, TargetSchemaConnectionDisposition.REUSABLE); + } + } + + private static TargetSchemaProvisioningException rollbackKnownFailure( + Connection connection, + MetadataDatabaseKind kind, + TargetSchemaProvisioningException original) { + boolean interrupted = Thread.interrupted(); + try { + try { + connection.rollback(); + } catch (SQLException | RuntimeException rollbackFailure) { + return failure(kind, TargetSchemaProvisioningFailure.Phase.ROLLBACK_OUTCOME_UNKNOWN, + rollbackFailure, TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + } + boolean restored = restoreAfterKnownFailure(connection); + TargetSchemaConnectionDisposition disposition = restored + && original.disposition() == TargetSchemaConnectionDisposition.REUSABLE + ? TargetSchemaConnectionDisposition.REUSABLE + : TargetSchemaConnectionDisposition.DISCARD_REQUIRED; + return new TargetSchemaProvisioningException(kind, original.failure(), disposition); + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + + private static boolean restoreAfterKnownSuccess(Connection connection) { + try { + connection.setAutoCommit(true); + return true; + } catch (SQLException | RuntimeException ignored) { + return false; + } + } + + private static boolean restoreAfterKnownFailure(Connection connection) { + return restoreAfterKnownSuccess(connection); + } + + private static void cleanupAfterFatal(Connection connection) { + boolean interrupted = Thread.interrupted(); + try { + try { + connection.rollback(); + } catch (SQLException | RuntimeException | Error ignored) { + return; + } + try { + connection.setAutoCommit(true); + } catch (SQLException | RuntimeException | Error ignored) { + // The caller always discards the connection after a fatal error. + } + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + + private static TargetSchemaProvisioningOutcome reusable() { + return new TargetSchemaProvisioningOutcome(TargetSchemaConnectionDisposition.REUSABLE); + } + + private static TargetSchemaProvisioningOutcome discardRequired() { + return new TargetSchemaProvisioningOutcome(TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + } + + private static TargetSchemaProvisioningException deadlineFailure( + MetadataDatabaseKind kind, + TargetSchemaConnectionDisposition disposition) { + return failure(kind, TargetSchemaProvisioningFailure.Phase.DEADLINE, null, disposition); + } + + private static TargetSchemaProvisioningException failure( + MetadataDatabaseKind kind, + TargetSchemaProvisioningFailure.Phase phase, + Throwable cause, + TargetSchemaConnectionDisposition disposition) { + return new TargetSchemaProvisioningException( + kind, TargetSchemaProvisioningFailure.from(phase, cause), disposition); + } + + private static MetadataDatabaseKind supportedKind(MetadataDatabaseKind kind) { + return switch (Objects.requireNonNull(kind, "target kind")) { + case MYSQL -> MetadataDatabaseKind.MYSQL; + case POSTGRESQL -> MetadataDatabaseKind.POSTGRESQL; + case H2 -> throw new IllegalArgumentException("External target schema provisioning does not support H2"); + }; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioningWork.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioningWork.java new file mode 100644 index 0000000000..70fc6d7bac --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioningWork.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Applies the fixed baseline and its Flyway-compatible metadata under one exact JDBC budget. */ +final class FlywayTargetSchemaProvisioningWork implements TargetSchemaProvisioningWork { + + private static final String INSTALLED_BY = "hertzbeat-migration"; + private final MetadataDatabaseKind kind; + + FlywayTargetSchemaProvisioningWork(MetadataDatabaseKind kind) { + this.kind = kind; + } + + @Override + public void provision(Connection connection, TargetSchemaJdbcBudget budget) { + TargetSchemaBaseline baseline = loadBaseline(); + FlywaySchemaHistory history = new FlywaySchemaHistory(kind); + try { + budget.check(); + if (history.isCurrent(connection, baseline, budget)) { + return; + } + history.requireEmptyTarget(connection, budget); + } catch (SQLException failure) { + throw jdbcFailure(TargetSchemaProvisioningFailure.Phase.PRECONDITION, failure, + TargetSchemaConnectionDisposition.REUSABLE); + } catch (MetadataMigrationException timeout) { + throw deadlineFailure(TargetSchemaConnectionDisposition.REUSABLE); + } catch (RuntimeException unexpected) { + throw failure(TargetSchemaProvisioningFailure.Phase.PRECONDITION, unexpected, + TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + } + int executionTimeMillis = executeBaseline(connection, baseline, budget); + try { + history.record(connection, baseline, INSTALLED_BY, executionTimeMillis, budget); + } catch (SQLException failure) { + throw jdbcFailure(TargetSchemaProvisioningFailure.Phase.HISTORY_WRITE, failure, + TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + } catch (MetadataMigrationException timeout) { + throw deadlineFailure(TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + } catch (RuntimeException unexpected) { + throw failure(TargetSchemaProvisioningFailure.Phase.HISTORY_WRITE, unexpected, + TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + } + } + + private TargetSchemaBaseline loadBaseline() { + try { + return TargetSchemaBaseline.load(kind); + } catch (IOException failure) { + throw failure(TargetSchemaProvisioningFailure.Phase.BASELINE_RESOURCE, failure, + TargetSchemaConnectionDisposition.REUSABLE); + } catch (RuntimeException unexpected) { + throw failure(TargetSchemaProvisioningFailure.Phase.BASELINE_RESOURCE, unexpected, + TargetSchemaConnectionDisposition.REUSABLE); + } + } + + private int executeBaseline( + Connection connection, + TargetSchemaBaseline baseline, + TargetSchemaJdbcBudget budget) { + long startedAt = System.nanoTime(); + try (Statement statement = connection.createStatement()) { + for (String sql : baseline.statements()) { + budget.apply(statement); + statement.execute(sql); + } + } catch (SQLException failure) { + throw jdbcFailure(TargetSchemaProvisioningFailure.Phase.BASELINE_EXECUTION, failure, + TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + } catch (MetadataMigrationException timeout) { + throw deadlineFailure(TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + } catch (RuntimeException unexpected) { + throw failure(TargetSchemaProvisioningFailure.Phase.BASELINE_EXECUTION, unexpected, + TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + } + return Math.toIntExact(Math.min( + Integer.MAX_VALUE, (System.nanoTime() - startedAt) / 1_000_000L)); + } + + private TargetSchemaProvisioningException deadlineFailure( + TargetSchemaConnectionDisposition disposition) { + return failure(TargetSchemaProvisioningFailure.Phase.DEADLINE, null, disposition); + } + + private TargetSchemaProvisioningException jdbcFailure( + TargetSchemaProvisioningFailure.Phase phase, + SQLException cause, + TargetSchemaConnectionDisposition fallbackDisposition) { + if (TargetSchemaSqlFailure.isTimeout(cause)) { + return deadlineFailure(TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + } + TargetSchemaConnectionDisposition disposition = TargetSchemaSqlFailure.invalidatesConnection(cause) + ? TargetSchemaConnectionDisposition.DISCARD_REQUIRED + : fallbackDisposition; + return failure(phase, cause, disposition); + } + + private TargetSchemaProvisioningException failure( + TargetSchemaProvisioningFailure.Phase phase, + Throwable cause, + TargetSchemaConnectionDisposition disposition) { + return new TargetSchemaProvisioningException( + kind, TargetSchemaProvisioningFailure.from(phase, cause), disposition); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcTargetSchemaObjectState.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcTargetSchemaObjectState.java index 5392fa9afb..f10800b295 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcTargetSchemaObjectState.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcTargetSchemaObjectState.java @@ -36,11 +36,11 @@ final class JdbcTargetSchemaObjectState { Connection connection, MetadataDatabaseKind kind, Set baselineTables, - int queryTimeoutSeconds, + TargetSchemaJdbcBudget budget, FactSink facts) throws SQLException { - captureChecks(connection, kind, baselineTables, queryTimeoutSeconds, facts); - captureTriggers(connection, kind, baselineTables, queryTimeoutSeconds, facts); - captureSequences(connection, kind, queryTimeoutSeconds, facts); + captureChecks(connection, kind, baselineTables, budget, facts); + captureTriggers(connection, kind, baselineTables, budget, facts); + captureSequences(connection, kind, budget, facts); } static void captureIdentityOwnership( @@ -48,20 +48,22 @@ final class JdbcTargetSchemaObjectState { MetadataDatabaseKind kind, String table, List identities, - int queryTimeoutSeconds, + TargetSchemaJdbcBudget budget, FactSink facts) throws SQLException { if (kind != MetadataDatabaseKind.POSTGRESQL) { return; } try (PreparedStatement statement = connection.prepareStatement("SELECT pg_get_serial_sequence(?, ?)")) { - applyTimeout(statement, queryTimeoutSeconds); for (String column : identities) { + budget.apply(statement); statement.setString(1, table); statement.setString(2, column); try (ResultSet rows = statement.executeQuery()) { + budget.check(); if (!rows.next()) { throw new SQLException("Target identity sequence ownership is absent", "55000"); } + budget.check(); String sequence = rows.getString(1); if (sequence == null) { throw new SQLException("Target identity sequence ownership is absent", "55000"); @@ -69,6 +71,7 @@ final class JdbcTargetSchemaObjectState { facts.add("identity-sequence", table, column, normalize(sequence)); } } + budget.check(); } } @@ -76,7 +79,7 @@ final class JdbcTargetSchemaObjectState { Connection connection, MetadataDatabaseKind kind, Set baselineTables, - int queryTimeoutSeconds, + TargetSchemaJdbcBudget budget, FactSink facts) throws SQLException { String sql = kind == MetadataDatabaseKind.POSTGRESQL ? "SELECT relation.relname, pg_get_constraintdef(c.oid, false) " @@ -92,14 +95,17 @@ final class JdbcTargetSchemaObjectState { + "WHERE table_constraint.constraint_type = 'CHECK' " + "AND table_constraint.table_schema = DATABASE()"; try (PreparedStatement statement = connection.prepareStatement(sql)) { - applyTimeout(statement, queryTimeoutSeconds); + budget.apply(statement); try (ResultSet rows = statement.executeQuery()) { + budget.check(); while (rows.next()) { + budget.check(); String table = normalize(rows.getString(1)); if (baselineTables.contains(table)) { facts.add("check", table, rows.getString(2).strip()); } } + budget.check(); } } } @@ -108,7 +114,7 @@ final class JdbcTargetSchemaObjectState { Connection connection, MetadataDatabaseKind kind, Set baselineTables, - int queryTimeoutSeconds, + TargetSchemaJdbcBudget budget, FactSink facts) throws SQLException { String schemaPredicate = kind == MetadataDatabaseKind.POSTGRESQL ? "trigger_schema = current_schema()" @@ -116,15 +122,18 @@ final class JdbcTargetSchemaObjectState { String sql = "SELECT event_object_table, action_timing, event_manipulation, action_statement " + "FROM information_schema.triggers WHERE " + schemaPredicate; try (PreparedStatement statement = connection.prepareStatement(sql)) { - applyTimeout(statement, queryTimeoutSeconds); + budget.apply(statement); try (ResultSet rows = statement.executeQuery()) { + budget.check(); while (rows.next()) { + budget.check(); String table = normalize(rows.getString(1)); if (baselineTables.contains(table)) { facts.add("trigger", table, normalize(rows.getString(2)), normalize(rows.getString(3)), rows.getString(4).strip()); } } + budget.check(); } } } @@ -132,7 +141,7 @@ final class JdbcTargetSchemaObjectState { private static void captureSequences( Connection connection, MetadataDatabaseKind kind, - int queryTimeoutSeconds, + TargetSchemaJdbcBudget budget, FactSink facts) throws SQLException { if (kind != MetadataDatabaseKind.POSTGRESQL) { return; @@ -140,9 +149,11 @@ final class JdbcTargetSchemaObjectState { String sql = "SELECT schemaname, sequencename, increment_by, min_value, max_value, " + "cache_size, cycle FROM pg_sequences WHERE schemaname = current_schema()"; try (PreparedStatement statement = connection.prepareStatement(sql)) { - applyTimeout(statement, queryTimeoutSeconds); + budget.apply(statement); try (ResultSet rows = statement.executeQuery()) { + budget.check(); while (rows.next()) { + budget.check(); String qualifiedName = normalize(rows.getString(1)) + '.' + normalize(rows.getString(2)); facts.add( "sequence", @@ -153,16 +164,11 @@ final class JdbcTargetSchemaObjectState { "cache=" + rows.getLong(6), "cycle=" + rows.getBoolean(7)); } + budget.check(); } } } - private static void applyTimeout(PreparedStatement statement, int queryTimeoutSeconds) throws SQLException { - if (queryTimeoutSeconds > 0) { - statement.setQueryTimeout(queryTimeoutSeconds); - } - } - private static String normalize(String value) { return value.toLowerCase(Locale.ROOT); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcTargetSchemaState.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcTargetSchemaState.java index 94c3ae2792..9a356de0e5 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcTargetSchemaState.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/JdbcTargetSchemaState.java @@ -41,7 +41,7 @@ final class JdbcTargetSchemaState { Connection connection, MetadataDatabaseKind kind, Set baselineTables) throws SQLException { - return capture(connection, kind, baselineTables, 0); + return capture(connection, kind, baselineTables, TargetSchemaJdbcBudget.none()); } static SchemaState capture( @@ -49,21 +49,36 @@ final class JdbcTargetSchemaState { MetadataDatabaseKind kind, Set baselineTables, int queryTimeoutSeconds) throws SQLException { + return capture(connection, kind, baselineTables, TargetSchemaJdbcBudget.fixed(queryTimeoutSeconds)); + } + + static SchemaState capture( + Connection connection, + MetadataDatabaseKind kind, + Set baselineTables, + TargetSchemaJdbcBudget budget) throws SQLException { + budget.check(); DatabaseMetaData metadata = connection.getMetaData(); + budget.check(); String catalog = connection.getCatalog(); - String schema = kind == MetadataDatabaseKind.POSTGRESQL ? connection.getSchema() : null; + budget.check(); + String schema = null; + if (kind == MetadataDatabaseKind.POSTGRESQL) { + schema = connection.getSchema(); + budget.check(); + } FactCollector facts = new FactCollector(); for (String table : baselineTables.stream().sorted().toList()) { facts.add("table", table); - List identities = readColumns(metadata, catalog, schema, table, kind, facts); + List identities = readColumns(metadata, catalog, schema, table, kind, facts, budget); JdbcTargetSchemaObjectState.captureIdentityOwnership( - connection, kind, table, identities, queryTimeoutSeconds, facts::add); - readPrimaryKey(metadata, catalog, schema, table, facts); - readIndexes(metadata, catalog, schema, table, facts); - readForeignKeys(metadata, catalog, schema, table, facts); + connection, kind, table, identities, budget, facts::add); + readPrimaryKey(metadata, catalog, schema, table, facts, budget); + readIndexes(metadata, catalog, schema, table, facts, budget); + readForeignKeys(metadata, catalog, schema, table, facts, budget); } JdbcTargetSchemaObjectState.capture( - connection, kind, baselineTables, queryTimeoutSeconds, facts::add); + connection, kind, baselineTables, budget, facts::add); return facts.build(); } @@ -73,10 +88,14 @@ final class JdbcTargetSchemaState { String schema, String table, MetadataDatabaseKind kind, - FactCollector facts) throws SQLException { + FactCollector facts, + TargetSchemaJdbcBudget budget) throws SQLException { List identities = new ArrayList<>(); + budget.check(); try (ResultSet columns = metadata.getColumns(catalog, schema, table, null)) { + budget.check(); while (columns.next()) { + budget.check(); int jdbcType = columns.getInt("DATA_TYPE"); int size = columns.getInt("COLUMN_SIZE"); int scale = columns.getInt("DECIMAL_DIGITS"); @@ -95,6 +114,7 @@ final class JdbcTargetSchemaState { defaultValue(columns.getString("COLUMN_DEF"))); } } + budget.check(); return List.copyOf(identities); } @@ -103,13 +123,18 @@ final class JdbcTargetSchemaState { String catalog, String schema, String table, - FactCollector facts) throws SQLException { + FactCollector facts, + TargetSchemaJdbcBudget budget) throws SQLException { OrderedColumns columns = new OrderedColumns(); + budget.check(); try (ResultSet keys = metadata.getPrimaryKeys(catalog, schema, table)) { + budget.check(); while (keys.next()) { + budget.check(); columns.add(keys.getShort("KEY_SEQ"), normalize(keys.getString("COLUMN_NAME"))); } } + budget.check(); if (!columns.isEmpty()) { facts.add("primary-key", table, columns.definition()); } @@ -120,11 +145,15 @@ final class JdbcTargetSchemaState { String catalog, String schema, String table, - FactCollector facts) throws SQLException { + FactCollector facts, + TargetSchemaJdbcBudget budget) throws SQLException { Map indexes = new HashMap<>(); int unnamedIndex = 0; + budget.check(); try (ResultSet rows = metadata.getIndexInfo(catalog, schema, table, false, false)) { + budget.check(); while (rows.next()) { + budget.check(); String name = rows.getString("INDEX_NAME"); String column = rows.getString("COLUMN_NAME"); short position = rows.getShort("ORDINAL_POSITION"); @@ -141,6 +170,7 @@ final class JdbcTargetSchemaState { .add(position, normalize(column)); } } + budget.check(); indexes.values().forEach(index -> facts.add( "index", table, Boolean.toString(index.unique()), index.columns().definition())); } @@ -150,11 +180,15 @@ final class JdbcTargetSchemaState { String catalog, String schema, String table, - FactCollector facts) throws SQLException { + FactCollector facts, + TargetSchemaJdbcBudget budget) throws SQLException { Map keys = new HashMap<>(); int unnamedKey = 0; + budget.check(); try (ResultSet rows = metadata.getImportedKeys(catalog, schema, table)) { + budget.check(); while (rows.next()) { + budget.check(); String referencedTable = normalize(rows.getString("PKTABLE_NAME")); String name = rows.getString("FK_NAME"); short position = rows.getShort("KEY_SEQ"); @@ -174,6 +208,7 @@ final class JdbcTargetSchemaState { normalize(rows.getString("PKCOLUMN_NAME"))); } } + budget.check(); keys.values().forEach(key -> facts.add( "foreign-key", table, key.localColumns(), key.referencedTable(), key.referencedColumns(), key.updateRule(), key.deleteRule(), key.deferrability())); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaConnectionDisposition.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaConnectionDisposition.java new file mode 100644 index 0000000000..993292d997 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaConnectionDisposition.java @@ -0,0 +1,14 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +/** Tells the connection owner whether a target connection can be reused after provisioning. */ +public enum TargetSchemaConnectionDisposition { + REUSABLE, + DISCARD_REQUIRED +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaContract.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaContract.java index f9fec6166d..7ef879212a 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaContract.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaContract.java @@ -42,44 +42,63 @@ final class TargetSchemaContract { } void record(Connection connection, Set baselineTables) throws SQLException { - JdbcTargetSchemaState.SchemaState state = JdbcTargetSchemaState.capture(connection, kind, baselineTables); + record(connection, baselineTables, TargetSchemaJdbcBudget.none()); + } + + void record( + Connection connection, + Set baselineTables, + TargetSchemaJdbcBudget budget) throws SQLException { + JdbcTargetSchemaState.SchemaState state = + JdbcTargetSchemaState.capture(connection, kind, baselineTables, budget); try (Statement statement = connection.createStatement()) { + budget.apply(statement); statement.execute(CREATE_TABLE); } String insert = "INSERT INTO " + TABLE + " (contract_id, database_kind, definition, occurrences) VALUES (?, ?, ?, ?)"; try (PreparedStatement statement = connection.prepareStatement(insert)) { + budget.apply(statement); int contractId = 1; for (Map.Entry fact : state.facts().entrySet()) { + budget.check(); statement.setInt(1, contractId++); statement.setString(2, kind.name()); statement.setString(3, fact.getKey()); statement.setInt(4, fact.getValue()); statement.addBatch(); } + budget.apply(statement); statement.executeBatch(); } } boolean matches(Connection connection, Set baselineTables) throws SQLException { - return matches(connection, baselineTables, 0); + return matches(connection, baselineTables, TargetSchemaJdbcBudget.none()); } boolean matches(Connection connection, Set baselineTables, int queryTimeoutSeconds) throws SQLException { - return JdbcTargetSchemaState.capture(connection, kind, baselineTables, queryTimeoutSeconds) - .equals(readRecordedState(connection, queryTimeoutSeconds)); + return matches(connection, baselineTables, TargetSchemaJdbcBudget.fixed(queryTimeoutSeconds)); + } + + boolean matches( + Connection connection, + Set baselineTables, + TargetSchemaJdbcBudget budget) throws SQLException { + return JdbcTargetSchemaState.capture(connection, kind, baselineTables, budget) + .equals(readRecordedState(connection, budget)); } private JdbcTargetSchemaState.SchemaState readRecordedState( - Connection connection, int queryTimeoutSeconds) throws SQLException { + Connection connection, TargetSchemaJdbcBudget budget) throws SQLException { Map facts = new TreeMap<>(); String select = "SELECT database_kind, definition, occurrences FROM " + TABLE; try (PreparedStatement statement = connection.prepareStatement(select)) { - if (queryTimeoutSeconds > 0) { - statement.setQueryTimeout(queryTimeoutSeconds); - } + budget.apply(statement); try (ResultSet rows = statement.executeQuery()) { + budget.check(); while (rows.next()) { + budget.check(); if (!kind.name().equals(rows.getString("database_kind"))) { throw new SQLException("Target schema contract contains another database kind", "55000"); } @@ -88,6 +107,7 @@ final class TargetSchemaContract { throw new SQLException("Target schema contract contains duplicate definitions", "55000"); } } + budget.check(); } } return new JdbcTargetSchemaState.SchemaState(facts); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaJdbcBudget.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaJdbcBudget.java new file mode 100644 index 0000000000..3d0ee250b5 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaJdbcBudget.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +/** Applies one exact monotonic deadline to cooperative JDBC schema operations. */ +final class TargetSchemaJdbcBudget { + + private final JdbcMetadataMigrationDeadline deadline; + private final int fixedSeconds; + + TargetSchemaJdbcBudget(JdbcMetadataMigrationDeadline deadline) { + this.deadline = Objects.requireNonNull(deadline, "deadline"); + this.fixedSeconds = 0; + } + + private TargetSchemaJdbcBudget(int fixedSeconds) { + this.deadline = null; + this.fixedSeconds = Math.max(0, fixedSeconds); + } + + static TargetSchemaJdbcBudget none() { + return new TargetSchemaJdbcBudget(0); + } + + static TargetSchemaJdbcBudget fixed(int seconds) { + return new TargetSchemaJdbcBudget(seconds); + } + + void check() { + if (Thread.currentThread().isInterrupted()) { + throw new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT); + } + if (deadline != null) { + deadline.remainingDuration(); + } + } + + void apply(Statement statement) throws SQLException { + Objects.requireNonNull(statement, "statement"); + check(); + if (deadline == null) { + if (fixedSeconds > 0) { + statement.setQueryTimeout(fixedSeconds); + } + check(); + return; + } + long remaining = deadline.remainingNanos(); + if (remaining <= 0) { + throw new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT); + } + long seconds = TimeUnit.NANOSECONDS.toSeconds(remaining); + if (remaining % TimeUnit.SECONDS.toNanos(1) != 0) { + seconds++; + } + statement.setQueryTimeout((int) Math.min(Integer.MAX_VALUE, Math.max(1, seconds))); + check(); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioningException.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioningException.java index 395c8543bc..fb549022eb 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioningException.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioningException.java @@ -23,14 +23,27 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseK public final class TargetSchemaProvisioningException extends RuntimeException { private final TargetSchemaProvisioningFailure failure; + private final TargetSchemaConnectionDisposition disposition; TargetSchemaProvisioningException( MetadataDatabaseKind kind, TargetSchemaProvisioningFailure failure) { + this(kind, failure, TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + } + + TargetSchemaProvisioningException( + MetadataDatabaseKind kind, + TargetSchemaProvisioningFailure failure, + TargetSchemaConnectionDisposition disposition) { super("Target schema provisioning failed for " + kind); this.failure = failure; + this.disposition = disposition; } public TargetSchemaProvisioningFailure failure() { return failure; } + + public TargetSchemaConnectionDisposition disposition() { + return disposition; + } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioningFailure.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioningFailure.java index 0ccc6c222d..9b7f72e26a 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioningFailure.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioningFailure.java @@ -72,6 +72,9 @@ public record TargetSchemaProvisioningFailure( BASELINE_EXECUTION, HISTORY_WRITE, TRANSACTION, - CLEANUP + CLEANUP, + DEADLINE, + COMMIT_OUTCOME_UNKNOWN, + ROLLBACK_OUTCOME_UNKNOWN } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioningOutcome.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioningOutcome.java new file mode 100644 index 0000000000..227ef36228 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioningOutcome.java @@ -0,0 +1,18 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; + +/** Secret-free successful schema provisioning result. */ +public record TargetSchemaProvisioningOutcome(TargetSchemaConnectionDisposition disposition) { + + public TargetSchemaProvisioningOutcome { + Objects.requireNonNull(disposition, "disposition"); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioningWork.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioningWork.java new file mode 100644 index 0000000000..21b805f70c --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisioningWork.java @@ -0,0 +1,17 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; + +/** Executes baseline and Flyway history work without owning the target connection. */ +@FunctionalInterface +interface TargetSchemaProvisioningWork { + + void provision(Connection connection, TargetSchemaJdbcBudget budget); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaSqlFailure.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaSqlFailure.java new file mode 100644 index 0000000000..6295819f23 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaSqlFailure.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.SQLException; +import java.sql.SQLNonTransientConnectionException; +import java.sql.SQLRecoverableException; +import java.sql.SQLTimeoutException; +import java.sql.SQLTransientConnectionException; + +/** Classifies JDBC failures without retaining driver messages or connection details. */ +final class TargetSchemaSqlFailure { + + private TargetSchemaSqlFailure() { + } + + static boolean isTimeout(SQLException failure) { + String state = failure.getSQLState(); + return failure instanceof SQLTimeoutException + || "HYT00".equals(state) + || "HYT01".equals(state) + || "57014".equals(state); + } + + static boolean invalidatesConnection(SQLException failure) { + String state = failure.getSQLState(); + return isTimeout(failure) + || failure instanceof SQLTransientConnectionException + || failure instanceof SQLNonTransientConnectionException + || failure instanceof SQLRecoverableException + || state != null && state.startsWith("08"); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaConnectionBudgetTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaConnectionBudgetTest.java new file mode 100644 index 0000000000..3af70df195 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaConnectionBudgetTest.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantLock; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +class FlywayTargetSchemaConnectionBudgetTest { + + @ParameterizedTest + @EnumSource(AdvanceAt.class) + void everyConnectionPreconditionCallIsFollowedByTheExactBudgetGate(AdvanceAt advanceAt) + throws Exception { + AtomicLong ticker = new AtomicLong(); + AtomicBoolean workEntered = new AtomicBoolean(); + Connection connection = mock(Connection.class); + DatabaseMetaData metadata = mock(DatabaseMetaData.class); + when(connection.getAutoCommit()).thenAnswer(invocation -> { + advance(advanceAt, AdvanceAt.AUTO_COMMIT, ticker); + return true; + }); + when(connection.isReadOnly()).thenAnswer(invocation -> { + advance(advanceAt, AdvanceAt.READ_ONLY, ticker); + return false; + }); + when(connection.getMetaData()).thenAnswer(invocation -> { + advance(advanceAt, AdvanceAt.METADATA, ticker); + return metadata; + }); + when(metadata.getDatabaseProductName()).thenAnswer(invocation -> { + advance(advanceAt, AdvanceAt.PRODUCT, ticker); + return "MySQL"; + }); + JdbcMetadataMigrationDeadline deadline = JdbcMetadataMigrationDeadline.start( + Duration.ofNanos(1), ticker::get); + FlywayTargetSchemaProvisioningCore core = new FlywayTargetSchemaProvisioningCore( + new ReentrantLock(), (actual, budget) -> workEntered.set(true)); + + assertThatThrownBy(() -> core.provision(connection, MetadataDatabaseKind.MYSQL, deadline)) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, failure -> { + assertThat(failure.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.DEADLINE); + assertThat(failure.disposition()) + .isEqualTo(TargetSchemaConnectionDisposition.REUSABLE); + }); + assertThat(workEntered.get()).isFalse(); + verifyNextCallWasGated(connection, metadata, advanceAt); + } + + @ParameterizedTest + @EnumSource(MetadataFailure.class) + void productMetadataFailureUsesPreconditionPhase(MetadataFailure metadataFailure) throws Exception { + Connection connection = mock(Connection.class); + DatabaseMetaData metadata = mock(DatabaseMetaData.class); + when(connection.getAutoCommit()).thenReturn(true); + when(connection.isReadOnly()).thenReturn(false); + if (metadataFailure == MetadataFailure.LOOKUP) { + when(connection.getMetaData()).thenThrow(new SQLException("private metadata", "08006")); + } else { + when(connection.getMetaData()).thenReturn(metadata); + when(metadata.getDatabaseProductName()).thenThrow(new SQLException("private product", "08006")); + } + + assertThatThrownBy(() -> new FlywayTargetSchemaProvisioningCore( + new ReentrantLock(), (actual, budget) -> { }) + .provision(connection, MetadataDatabaseKind.MYSQL, deadline())) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, failure -> { + assertThat(failure.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.PRECONDITION); + assertThat(failure.disposition()) + .isEqualTo(TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + }); + } + + @ParameterizedTest + @EnumSource(StateFailure.class) + void connectionStateFailureUsesTransactionPhase(StateFailure stateFailure) throws Exception { + Connection connection = mock(Connection.class); + if (stateFailure == StateFailure.AUTO_COMMIT) { + when(connection.getAutoCommit()).thenThrow(new SQLException("private auto-commit", "08006")); + } else { + when(connection.getAutoCommit()).thenReturn(true); + when(connection.isReadOnly()).thenThrow(new SQLException("private read-only", "08006")); + } + + assertThatThrownBy(() -> new FlywayTargetSchemaProvisioningCore( + new ReentrantLock(), (actual, budget) -> { }) + .provision(connection, MetadataDatabaseKind.MYSQL, deadline())) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, failure -> { + assertThat(failure.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.TRANSACTION); + assertThat(failure.disposition()) + .isEqualTo(TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + }); + } + + private static void advance(AdvanceAt actual, AdvanceAt expected, AtomicLong ticker) { + if (actual == expected) { + ticker.set(2); + } + } + + private static void verifyNextCallWasGated( + Connection connection, + DatabaseMetaData metadata, + AdvanceAt advanceAt) throws Exception { + switch (advanceAt) { + case AUTO_COMMIT -> verify(connection, never()).isReadOnly(); + case READ_ONLY -> verify(connection, never()).getMetaData(); + case METADATA -> verify(metadata, never()).getDatabaseProductName(); + case PRODUCT -> { } + default -> throw new AssertionError("Unexpected connection precondition step"); + } + } + + private static JdbcMetadataMigrationDeadline deadline() { + return JdbcMetadataMigrationDeadline.start(Duration.ofSeconds(5), System::nanoTime); + } + + private enum AdvanceAt { + AUTO_COMMIT, + READ_ONLY, + METADATA, + PRODUCT + } + + private enum MetadataFailure { + LOOKUP, + PRODUCT_NAME + } + + private enum StateFailure { + AUTO_COMMIT, + READ_ONLY + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisionerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisionerTest.java index c9b1dcb6df..4fb9fae179 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisionerTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisionerTest.java @@ -19,6 +19,10 @@ package org.apache.hertzbeat.manager.setup.workflow; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import java.lang.reflect.Proxy; import java.sql.Connection; @@ -26,6 +30,7 @@ import java.sql.Driver; import java.sql.DriverManager; import java.sql.DriverPropertyInfo; import java.sql.SQLException; +import java.time.Duration; import java.util.List; import java.util.Properties; import java.util.concurrent.CopyOnWriteArrayList; @@ -47,6 +52,24 @@ class FlywayTargetSchemaProvisionerTest { private static final String FLYWAY_LOG_FACTORY = "flyway-log-factory"; + @Test + void callerOwnedEntryAcceptsNoIdentityAndNeverClosesOnDeadline() throws Exception { + Connection connection = mock(Connection.class); + long[] ticks = {0, 2}; + int[] index = {0}; + JdbcMetadataMigrationDeadline deadline = JdbcMetadataMigrationDeadline.start( + Duration.ofNanos(1), () -> ticks[Math.min(index[0]++, ticks.length - 1)]); + + assertThatThrownBy(() -> new FlywayTargetSchemaProvisioner() + .provision(connection, MetadataDatabaseKind.MYSQL, deadline)) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, failure -> { + assertThat(failure.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.DEADLINE); + assertThat(failure.disposition()).isEqualTo(TargetSchemaConnectionDisposition.REUSABLE); + }); + verify(connection, never()).close(); + } + @Test void rejectsEmbeddedTargetsBeforeConnectionOpen() { MetadataDatabaseConfiguration target = new MetadataDatabaseConfiguration( @@ -109,7 +132,7 @@ class FlywayTargetSchemaProvisionerTest { assertThatThrownBy(() -> new FlywayTargetSchemaProvisioner().provision(target)) .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, exception -> { assertThat(exception.failure().phase()) - .isEqualTo(TargetSchemaProvisioningFailure.Phase.BASELINE_RESOURCE); + .isEqualTo(TargetSchemaProvisioningFailure.Phase.PRECONDITION); assertThat(exception.getSuppressed()).isEmpty(); assertThat(exception.getMessage()).doesNotContain(jdbcUrl, "secret-value", "SELECT"); }); @@ -118,6 +141,56 @@ class FlywayTargetSchemaProvisionerTest { } } + @Test + void ownedConnectionCloseFailureAfterSuccessIsStableCleanupFailure() throws Exception { + Connection connection = mock(Connection.class); + doThrow(new SQLException("private close diagnostic", "08006", 93)).when(connection).close(); + + assertThatThrownBy(() -> new FlywayTargetSchemaProvisioner() + .provisionOwned(connection, MetadataDatabaseKind.MYSQL, () -> { })) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, failure -> { + assertThat(failure.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.CLEANUP); + assertThat(failure).hasNoCause(); + assertThat(failure.getMessage()).doesNotContain("private close diagnostic"); + }); + } + + @Test + void ownedConnectionCloseErrorNeverOverridesStableOperationFailure() throws Exception { + Connection connection = mock(Connection.class); + doThrow(new AssertionError("private close fatal")).when(connection).close(); + TargetSchemaProvisioningException operation = new TargetSchemaProvisioningException( + MetadataDatabaseKind.MYSQL, + new TargetSchemaProvisioningFailure( + TargetSchemaProvisioningFailure.Phase.PRECONDITION, + TargetSchemaBaseline.VERSION, + null, + 0), + TargetSchemaConnectionDisposition.REUSABLE); + + assertThatThrownBy(() -> new FlywayTargetSchemaProvisioner() + .provisionOwned(connection, MetadataDatabaseKind.MYSQL, () -> { + throw operation; + })) + .isSameAs(operation) + .satisfies(failure -> assertThat(failure.getSuppressed()).isEmpty()); + } + + @Test + void ownedConnectionCloseErrorNeverOverridesEarlierFatal() throws Exception { + Connection connection = mock(Connection.class); + doThrow(new AssertionError("private close fatal")).when(connection).close(); + AssertionError operation = new AssertionError("first fatal"); + + assertThatThrownBy(() -> new FlywayTargetSchemaProvisioner() + .provisionOwned(connection, MetadataDatabaseKind.MYSQL, () -> { + throw operation; + })) + .isSameAs(operation) + .satisfies(failure -> assertThat(failure.getSuppressed()).isEmpty()); + } + @Test @ResourceLock(FLYWAY_LOG_FACTORY) void provisioningDoesNotReplaceLoggerUsedByAnInterleavedFlywayOperation() throws Exception { @@ -257,6 +330,15 @@ class FlywayTargetSchemaProvisionerTest { if (method.getName().equals("close")) { throw new SQLException("close leaked " + url + " after SELECT secret-value", "08006", 999); } + if (method.getName().equals("getAutoCommit")) { + return true; + } + if (method.getName().equals("isReadOnly")) { + return false; + } + if (method.getName().equals("getMetaData")) { + throw new SQLException("metadata unavailable after SELECT secret-value", "08006", 998); + } return null; }); } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioningCoreTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioningCoreTest.java new file mode 100644 index 0000000000..a5587420c7 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioningCoreTest.java @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +class FlywayTargetSchemaProvisioningCoreTest { + + @Test + void mysqlUsesExactCallerConnectionWithoutOwningItsLifecycle() throws Exception { + Connection connection = idleConnection(MetadataDatabaseKind.MYSQL); + FlywayTargetSchemaProvisioningCore core = core((actual, budget) -> { + assertThat(actual).isSameAs(connection); + budget.check(); + }); + + TargetSchemaProvisioningOutcome outcome = core.provision( + connection, MetadataDatabaseKind.MYSQL, deadline(Duration.ofSeconds(5))); + + assertThat(outcome.disposition()).isEqualTo(TargetSchemaConnectionDisposition.REUSABLE); + verify(connection, never()).setAutoCommit(false); + verify(connection, never()).commit(); + verify(connection, never()).rollback(); + verify(connection, never()).close(); + } + + @Test + void postgresqlRestoresCallerStateAfterKnownCommit() throws Exception { + Connection connection = postgresConnection(); + + TargetSchemaProvisioningOutcome outcome = core((actual, budget) -> budget.check()) + .provision(connection, MetadataDatabaseKind.POSTGRESQL, deadline(Duration.ofSeconds(5))); + + assertThat(outcome.disposition()).isEqualTo(TargetSchemaConnectionDisposition.REUSABLE); + verify(connection).setAutoCommit(false); + verify(connection).commit(); + verify(connection).setAutoCommit(true); + verify(connection, never()).rollback(); + verify(connection, never()).close(); + } + + @Test + void postgresqlCommitFailureIsOutcomeUnknownAndNeverRollsBack() throws Exception { + Connection connection = postgresConnection(); + doThrow(new SQLException("private commit diagnostic", "08006", 91)).when(connection).commit(); + + assertThatThrownBy(() -> core((actual, budget) -> { }) + .provision(connection, MetadataDatabaseKind.POSTGRESQL, deadline(Duration.ofSeconds(5)))) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, failure -> { + assertThat(failure.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.COMMIT_OUTCOME_UNKNOWN); + assertThat(failure.disposition()) + .isEqualTo(TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + assertThat(failure).hasNoCause(); + }); + verify(connection, never()).rollback(); + verify(connection, never()).setAutoCommit(true); + verify(connection, never()).close(); + } + + @Test + void postgresqlKnownRollbackPreservesOriginalFailureAndRestoresState() throws Exception { + Connection connection = postgresConnection(); + TargetSchemaProvisioningException original = failure( + TargetSchemaProvisioningFailure.Phase.BASELINE_EXECUTION, + TargetSchemaConnectionDisposition.REUSABLE); + + assertThatThrownBy(() -> core((actual, budget) -> { + throw original; + }).provision(connection, MetadataDatabaseKind.POSTGRESQL, deadline(Duration.ofSeconds(5)))) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, failure -> { + assertThat(failure.failure()).isEqualTo(original.failure()); + assertThat(failure.disposition()).isEqualTo(TargetSchemaConnectionDisposition.REUSABLE); + }); + verify(connection).rollback(); + verify(connection).setAutoCommit(true); + verify(connection, never()).close(); + } + + @Test + void postgresqlRollbackFailureOutranksOperationFailure() throws Exception { + Connection connection = postgresConnection(); + doThrow(new SQLException("private rollback diagnostic", "08006", 92)).when(connection).rollback(); + + assertThatThrownBy(() -> core((actual, budget) -> { + throw failure(TargetSchemaProvisioningFailure.Phase.HISTORY_WRITE, + TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + }).provision(connection, MetadataDatabaseKind.POSTGRESQL, deadline(Duration.ofSeconds(5)))) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, failure -> { + assertThat(failure.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.ROLLBACK_OUTCOME_UNKNOWN); + assertThat(failure.disposition()) + .isEqualTo(TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + assertThat(failure).hasNoCause(); + }); + verify(connection, never()).setAutoCommit(true); + verify(connection, never()).close(); + } + + @Test + void postCommitStateRestoreFailureDoesNotTurnKnownSchemaSuccessIntoFailure() throws Exception { + Connection connection = postgresConnection(); + doThrow(new SQLException("private restore diagnostic", "08006", 93)) + .when(connection).setAutoCommit(true); + + TargetSchemaProvisioningOutcome outcome = core((actual, budget) -> { }) + .provision(connection, MetadataDatabaseKind.POSTGRESQL, deadline(Duration.ofSeconds(5))); + + assertThat(outcome.disposition()) + .isEqualTo(TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + verify(connection).commit(); + verify(connection, never()).close(); + } + + @Test + void mysqlMutationFailureIsFailClosedWithoutTransactionCompensation() throws Exception { + Connection connection = idleConnection(MetadataDatabaseKind.MYSQL); + + assertThatThrownBy(() -> core((actual, budget) -> { + throw failure(TargetSchemaProvisioningFailure.Phase.BASELINE_EXECUTION, + TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + }).provision(connection, MetadataDatabaseKind.MYSQL, deadline(Duration.ofSeconds(5)))) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, failure -> + assertThat(failure.disposition()) + .isEqualTo(TargetSchemaConnectionDisposition.DISCARD_REQUIRED)); + verify(connection, never()).commit(); + verify(connection, never()).rollback(); + verify(connection, never()).setAutoCommit(false); + verify(connection, never()).close(); + } + + @Test + void exactDeadlineFailureHasStablePhaseAndLeavesUntouchedConnectionReusable() { + Connection connection = idleConnection(MetadataDatabaseKind.MYSQL); + + assertThatThrownBy(() -> core((actual, budget) -> budget.check()) + .provision(connection, MetadataDatabaseKind.MYSQL, expiredDeadline())) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, failure -> { + assertThat(failure.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.DEADLINE); + assertThat(failure.disposition()).isEqualTo(TargetSchemaConnectionDisposition.REUSABLE); + }); + } + + @Test + @Timeout(5) + void lockWaitConsumesTheExactDeadlineAndNeverRunsSchemaWork() throws Exception { + ReentrantLock lock = new ReentrantLock(); + CountDownLatch locked = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + try (ExecutorService worker = Executors.newSingleThreadExecutor()) { + Future owner = worker.submit(() -> { + lock.lock(); + try { + locked.countDown(); + release.await(); + } finally { + lock.unlock(); + } + return null; + }); + assertThat(locked.await(2, TimeUnit.SECONDS)).isTrue(); + Connection connection = mock(Connection.class); + FlywayTargetSchemaProvisioningCore core = new FlywayTargetSchemaProvisioningCore( + lock, (actual, budget) -> { + throw new AssertionError("schema work must not run after lock timeout"); + }); + try { + assertThatThrownBy(() -> core.provision( + connection, MetadataDatabaseKind.MYSQL, deadline(Duration.ofMillis(20)))) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, failure -> { + assertThat(failure.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.DEADLINE); + assertThat(failure.disposition()) + .isEqualTo(TargetSchemaConnectionDisposition.REUSABLE); + }); + } finally { + release.countDown(); + } + owner.get(2, TimeUnit.SECONDS); + } + } + + @Test + void interruptedLockWaitRestoresCallerInterruptStatus() { + ReentrantLock lock = new ReentrantLock(); + try { + Thread.currentThread().interrupt(); + assertThatThrownBy(() -> new FlywayTargetSchemaProvisioningCore(lock, (actual, budget) -> { }) + .provision(mock(Connection.class), MetadataDatabaseKind.MYSQL, + deadline(Duration.ofSeconds(5)))) + .isInstanceOf(TargetSchemaProvisioningException.class); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } finally { + Thread.interrupted(); + } + } + + private static FlywayTargetSchemaProvisioningCore core(TargetSchemaProvisioningWork work) { + return new FlywayTargetSchemaProvisioningCore(new ReentrantLock(), work); + } + + private static Connection postgresConnection() { + return idleConnection(MetadataDatabaseKind.POSTGRESQL); + } + + private static Connection idleConnection(MetadataDatabaseKind kind) { + try { + Connection connection = mock(Connection.class); + DatabaseMetaData metadata = mock(DatabaseMetaData.class); + when(connection.getAutoCommit()).thenReturn(true); + when(connection.isReadOnly()).thenReturn(false); + when(connection.getMetaData()).thenReturn(metadata); + when(metadata.getDatabaseProductName()).thenReturn(switch (kind) { + case MYSQL -> "MySQL"; + case POSTGRESQL -> "PostgreSQL"; + case H2 -> "H2"; + }); + return connection; + } catch (Exception failure) { + throw new AssertionError(failure); + } + } + + private static JdbcMetadataMigrationDeadline deadline(Duration duration) { + return JdbcMetadataMigrationDeadline.start(duration, System::nanoTime); + } + + private static JdbcMetadataMigrationDeadline expiredDeadline() { + long[] ticks = {0, 2}; + int[] index = {0}; + return JdbcMetadataMigrationDeadline.start( + Duration.ofNanos(1), () -> ticks[Math.min(index[0]++, ticks.length - 1)]); + } + + private static TargetSchemaProvisioningException failure( + TargetSchemaProvisioningFailure.Phase phase, + TargetSchemaConnectionDisposition disposition) { + return new TargetSchemaProvisioningException( + MetadataDatabaseKind.MYSQL, + new TargetSchemaProvisioningFailure(phase, TargetSchemaBaseline.VERSION, null, 0), + disposition); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioningSafetyTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioningSafetyTest.java new file mode 100644 index 0000000000..654d25dabd --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioningSafetyTest.java @@ -0,0 +1,280 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +class FlywayTargetSchemaProvisioningSafetyTest { + + @Test + void mysqlRequiresIdleWritableConnectionBeforeWork() throws Exception { + Connection connection = idleConnection(MetadataDatabaseKind.MYSQL); + when(connection.isReadOnly()).thenReturn(true); + + assertThatThrownBy(() -> core((actual, budget) -> { + throw new AssertionError("read-only target must not run schema work"); + }).provision(connection, MetadataDatabaseKind.MYSQL, deadline(Duration.ofSeconds(5)))) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, failure -> { + assertThat(failure.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.TRANSACTION); + assertThat(failure.disposition()).isEqualTo(TargetSchemaConnectionDisposition.REUSABLE); + }); + verify(connection).getAutoCommit(); + verify(connection).isReadOnly(); + verify(connection, never()).commit(); + verify(connection, never()).rollback(); + } + + @Test + void timeoutReportedByActiveWorkRemainsDiscardRequired() throws Exception { + Connection connection = idleConnection(MetadataDatabaseKind.MYSQL); + + assertThatThrownBy(() -> core((actual, budget) -> { + throw failure(TargetSchemaProvisioningFailure.Phase.DEADLINE, + TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + }).provision(connection, MetadataDatabaseKind.MYSQL, deadline(Duration.ofSeconds(5)))) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, failure -> + assertThat(failure.disposition()) + .isEqualTo(TargetSchemaConnectionDisposition.DISCARD_REQUIRED)); + } + + @Test + void mysqlTimeoutRaisedAfterWorkStartsRequiresDiscard() throws Exception { + Connection connection = idleConnection(MetadataDatabaseKind.MYSQL); + + assertThatThrownBy(() -> core((actual, budget) -> { + throw new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT); + }).provision(connection, MetadataDatabaseKind.MYSQL, deadline(Duration.ofSeconds(5)))) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, failure -> { + assertThat(failure.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.DEADLINE); + assertThat(failure.disposition()) + .isEqualTo(TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + }); + } + + @Test + void productMismatchIsRejectedBeforeSchemaWork() throws Exception { + Connection connection = idleConnection(MetadataDatabaseKind.POSTGRESQL); + + assertThatThrownBy(() -> core((actual, budget) -> { + throw new AssertionError("mismatched target must not run schema work"); + }).provision(connection, MetadataDatabaseKind.MYSQL, deadline(Duration.ofSeconds(5)))) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, failure -> { + assertThat(failure.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.PRECONDITION); + assertThat(failure.disposition()) + .isEqualTo(TargetSchemaConnectionDisposition.REUSABLE); + }); + verify(connection, never()).commit(); + verify(connection, never()).rollback(); + } + + @Test + void rollbackAndRestoreNeverDowngradeDiscardRequiredFailure() throws Exception { + Connection connection = idleConnection(MetadataDatabaseKind.POSTGRESQL); + + assertThatThrownBy(() -> core((actual, budget) -> { + throw failure(TargetSchemaProvisioningFailure.Phase.BASELINE_EXECUTION, + TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + }).provision(connection, MetadataDatabaseKind.POSTGRESQL, deadline(Duration.ofSeconds(5)))) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, failure -> + assertThat(failure.disposition()) + .isEqualTo(TargetSchemaConnectionDisposition.DISCARD_REQUIRED)); + verify(connection).rollback(); + verify(connection).setAutoCommit(true); + } + + @Test + void unexpectedWorkRuntimeIsSanitizedAndRolledBack() throws Exception { + Connection connection = idleConnection(MetadataDatabaseKind.POSTGRESQL); + + assertThatThrownBy(() -> core((actual, budget) -> { + throw new IllegalStateException("private runtime diagnostic"); + }).provision(connection, MetadataDatabaseKind.POSTGRESQL, deadline(Duration.ofSeconds(5)))) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, failure -> { + assertThat(failure.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.BASELINE_EXECUTION); + assertThat(failure.disposition()) + .isEqualTo(TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + assertThat(failure).hasNoCause(); + assertThat(failure.getMessage()).doesNotContain("private runtime diagnostic"); + }); + verify(connection).rollback(); + } + + @Test + void commitRuntimeIsSanitizedAsOutcomeUnknown() throws Exception { + Connection connection = idleConnection(MetadataDatabaseKind.POSTGRESQL); + doThrow(new IllegalStateException("private commit runtime")).when(connection).commit(); + + assertThatThrownBy(() -> core((actual, budget) -> { }) + .provision(connection, MetadataDatabaseKind.POSTGRESQL, deadline(Duration.ofSeconds(5)))) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, failure -> { + assertThat(failure.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.COMMIT_OUTCOME_UNKNOWN); + assertThat(failure.disposition()) + .isEqualTo(TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + assertThat(failure).hasNoCause(); + }); + verify(connection, never()).rollback(); + } + + @Test + void rollbackRuntimeIsSanitizedAsOutcomeUnknown() throws Exception { + Connection connection = idleConnection(MetadataDatabaseKind.POSTGRESQL); + doThrow(new IllegalStateException("private rollback runtime")).when(connection).rollback(); + + assertThatThrownBy(() -> core((actual, budget) -> { + throw failure(TargetSchemaProvisioningFailure.Phase.HISTORY_WRITE, + TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + }).provision(connection, MetadataDatabaseKind.POSTGRESQL, deadline(Duration.ofSeconds(5)))) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, failure -> { + assertThat(failure.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.ROLLBACK_OUTCOME_UNKNOWN); + assertThat(failure.disposition()) + .isEqualTo(TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + assertThat(failure).hasNoCause(); + }); + } + + @Test + void interruptIsClearedForRollbackAndRestoredForCaller() throws Exception { + Connection connection = idleConnection(MetadataDatabaseKind.POSTGRESQL); + doAnswer(invocation -> { + assertThat(Thread.currentThread().isInterrupted()).isFalse(); + return null; + }).when(connection).rollback(); + try { + assertThatThrownBy(() -> core((actual, budget) -> { + Thread.currentThread().interrupt(); + budget.check(); + }).provision(connection, MetadataDatabaseKind.POSTGRESQL, deadline(Duration.ofSeconds(5)))) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, failure -> + assertThat(failure.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.DEADLINE)); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } finally { + Thread.interrupted(); + } + } + + @Test + void deadlineIsCheckedAfterWorkBeforePostgresqlCommit() throws Exception { + Connection connection = idleConnection(MetadataDatabaseKind.POSTGRESQL); + long[] ticks = {0, 0, 0, 0, 0, 0, 0, 0, 0, 2}; + int[] index = {0}; + JdbcMetadataMigrationDeadline deadline = JdbcMetadataMigrationDeadline.start( + Duration.ofNanos(1), () -> ticks[Math.min(index[0]++, ticks.length - 1)]); + + assertThatThrownBy(() -> core((actual, budget) -> { }) + .provision(connection, MetadataDatabaseKind.POSTGRESQL, deadline)) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, failure -> + assertThat(failure.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.DEADLINE)); + verify(connection).rollback(); + verify(connection, never()).commit(); + } + + @Test + @Timeout(5) + void defaultCoreInstancesShareOneAdmissionLock() throws Exception { + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + FlywayTargetSchemaProvisioningCore first = + new FlywayTargetSchemaProvisioningCore((actual, budget) -> { + entered.countDown(); + await(release); + }); + FlywayTargetSchemaProvisioningCore second = + new FlywayTargetSchemaProvisioningCore((actual, budget) -> { + throw new AssertionError("second core must not pass the shared lock"); + }); + try (ExecutorService worker = Executors.newSingleThreadExecutor()) { + Future active = worker.submit(() -> first.provision( + idleConnection(MetadataDatabaseKind.MYSQL), MetadataDatabaseKind.MYSQL, + deadline(Duration.ofSeconds(5)))); + assertThat(entered.await(2, TimeUnit.SECONDS)).isTrue(); + try { + assertThatThrownBy(() -> second.provision( + idleConnection(MetadataDatabaseKind.MYSQL), MetadataDatabaseKind.MYSQL, + deadline(Duration.ofMillis(20)))) + .isInstanceOfSatisfying(TargetSchemaProvisioningException.class, failure -> + assertThat(failure.failure().phase()) + .isEqualTo(TargetSchemaProvisioningFailure.Phase.DEADLINE)); + } finally { + release.countDown(); + } + active.get(2, TimeUnit.SECONDS); + } + } + + private static FlywayTargetSchemaProvisioningCore core(TargetSchemaProvisioningWork work) { + return new FlywayTargetSchemaProvisioningCore(work); + } + + private static Connection idleConnection(MetadataDatabaseKind kind) { + try { + Connection connection = mock(Connection.class); + DatabaseMetaData metadata = mock(DatabaseMetaData.class); + when(connection.getAutoCommit()).thenReturn(true); + when(connection.isReadOnly()).thenReturn(false); + when(connection.getMetaData()).thenReturn(metadata); + when(metadata.getDatabaseProductName()).thenReturn(switch (kind) { + case MYSQL -> "MySQL"; + case POSTGRESQL -> "PostgreSQL"; + case H2 -> "H2"; + }); + return connection; + } catch (SQLException failure) { + throw new AssertionError(failure); + } + } + + private static JdbcMetadataMigrationDeadline deadline(Duration timeout) { + return JdbcMetadataMigrationDeadline.start(timeout, System::nanoTime); + } + + private static TargetSchemaProvisioningException failure( + TargetSchemaProvisioningFailure.Phase phase, + TargetSchemaConnectionDisposition disposition) { + return new TargetSchemaProvisioningException( + MetadataDatabaseKind.POSTGRESQL, + new TargetSchemaProvisioningFailure(phase, TargetSchemaBaseline.VERSION, null, 0), + disposition); + } + + private static void await(CountDownLatch latch) { + try { + latch.await(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new AssertionError(interrupted); + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioningWorkTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioningWorkTest.java new file mode 100644 index 0000000000..56c41c3aac --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FlywayTargetSchemaProvisioningWorkTest.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowableOfType; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLRecoverableException; +import java.sql.SQLTimeoutException; +import java.sql.SQLTransientConnectionException; +import java.sql.Statement; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.MockedStatic; + +class FlywayTargetSchemaProvisioningWorkTest { + + @ParameterizedTest + @MethodSource("connectionFailures") + void preconditionConnectionFailureRequiresDiscard(SQLException diagnostic) throws Exception { + assertFailure(preconditionFailure(diagnostic), TargetSchemaProvisioningFailure.Phase.PRECONDITION); + } + + @ParameterizedTest + @MethodSource("timeoutFailures") + void preconditionTimeoutUsesStableDeadlinePhase(SQLException diagnostic) throws Exception { + assertFailure(preconditionFailure(diagnostic), TargetSchemaProvisioningFailure.Phase.DEADLINE); + } + + @Test + void baselineSqlTimeoutUsesStableDeadlinePhase() throws Exception { + Connection connection = emptyTarget(); + Statement statement = mock(Statement.class); + when(connection.createStatement()).thenReturn(statement); + when(statement.execute(anyString())) + .thenThrow(new SQLTimeoutException("private baseline timeout")); + + assertFailure(run(connection), TargetSchemaProvisioningFailure.Phase.DEADLINE); + } + + @Test + void unexpectedPreconditionRuntimeUsesPreconditionPhase() throws Exception { + Connection connection = mock(Connection.class); + when(connection.getMetaData()).thenThrow(new IllegalStateException("private precondition runtime")); + + assertFailure(run(connection), TargetSchemaProvisioningFailure.Phase.PRECONDITION); + } + + @Test + void unexpectedBaselineRuntimeUsesBaselineExecutionPhase() throws Exception { + Connection connection = emptyTarget(); + Statement statement = mock(Statement.class); + when(connection.createStatement()).thenReturn(statement); + when(statement.execute(anyString())) + .thenThrow(new IllegalStateException("private baseline runtime")); + + assertFailure(run(connection), TargetSchemaProvisioningFailure.Phase.BASELINE_EXECUTION); + } + + @Test + void unexpectedHistoryRuntimeUsesHistoryWritePhase() throws Exception { + assertFailure(run(historyFailure(new IllegalStateException("private history runtime"))), + TargetSchemaProvisioningFailure.Phase.HISTORY_WRITE); + } + + @Test + void historySqlTimeoutUsesStableDeadlinePhase() throws Exception { + assertFailure(run(historyFailure(new SQLTimeoutException("private history timeout"))), + TargetSchemaProvisioningFailure.Phase.DEADLINE); + } + + private static Connection historyFailure(Exception diagnostic) throws Exception { + Connection connection = emptyTarget(); + Statement baseline = mock(Statement.class); + Statement contract = mock(Statement.class); + Statement history = mock(Statement.class); + when(connection.createStatement()).thenReturn(baseline, contract, history); + when(history.execute(anyString())).thenThrow(diagnostic); + PreparedStatement checks = emptyQuery(); + PreparedStatement triggers = emptyQuery(); + PreparedStatement contractInsert = mock(PreparedStatement.class); + when(connection.prepareStatement(anyString())) + .thenReturn(checks, triggers, contractInsert); + return connection; + } + + private static TargetSchemaProvisioningException preconditionFailure(SQLException diagnostic) + throws Exception { + Connection connection = mock(Connection.class); + when(connection.getMetaData()).thenThrow(diagnostic); + return run(connection); + } + + private static TargetSchemaProvisioningException run(Connection connection) { + TargetSchemaBaseline baseline = mock(TargetSchemaBaseline.class); + when(baseline.statements()).thenReturn(List.of("CREATE TABLE test_target (id INT)")); + when(baseline.expectedTables()).thenReturn(Set.of()); + try (MockedStatic baselines = mockStatic(TargetSchemaBaseline.class)) { + baselines.when(() -> TargetSchemaBaseline.load(MetadataDatabaseKind.MYSQL)).thenReturn(baseline); + return catchThrowableOfType( + () -> new FlywayTargetSchemaProvisioningWork(MetadataDatabaseKind.MYSQL) + .provision(connection, TargetSchemaJdbcBudget.none()), + TargetSchemaProvisioningException.class); + } + } + + private static void assertFailure( + TargetSchemaProvisioningException failure, + TargetSchemaProvisioningFailure.Phase phase) { + assertThat(failure).isNotNull(); + assertThat(failure.failure().phase()).isEqualTo(phase); + assertThat(failure.disposition()).isEqualTo(TargetSchemaConnectionDisposition.DISCARD_REQUIRED); + assertThat(failure).hasNoCause(); + assertThat(failure.getMessage()).doesNotContain("private"); + } + + private static Connection emptyTarget() throws Exception { + Connection connection = mock(Connection.class); + DatabaseMetaData metadata = mock(DatabaseMetaData.class); + ResultSet tables = mock(ResultSet.class); + when(connection.getMetaData()).thenReturn(metadata); + when(metadata.getTables(any(), any(), anyString(), any())).thenReturn(tables); + return connection; + } + + private static PreparedStatement emptyQuery() throws Exception { + PreparedStatement statement = mock(PreparedStatement.class); + when(statement.executeQuery()).thenReturn(mock(ResultSet.class)); + return statement; + } + + private static Stream connectionFailures() { + return Stream.of( + Arguments.of(new SQLException("private transport", "08006")), + Arguments.of(new SQLRecoverableException("private recoverable")), + Arguments.of(new SQLTransientConnectionException("private transient"))); + } + + private static Stream timeoutFailures() { + return Stream.of( + Arguments.of(new SQLTimeoutException("private timeout")), + Arguments.of(new SQLException("private timeout", "HYT00")), + Arguments.of(new SQLException("private timeout", "HYT01")), + Arguments.of(new SQLException("private timeout", "57014"))); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaJdbcBudgetTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaJdbcBudgetTest.java new file mode 100644 index 0000000000..1ae08a841a --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaJdbcBudgetTest.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Statement; +import java.time.Duration; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.api.Test; + +class TargetSchemaJdbcBudgetTest { + + @Test + void statementReceivesPositiveCeilingOfExactRemainingBudget() throws Exception { + AtomicLong ticker = new AtomicLong(); + TargetSchemaJdbcBudget budget = new TargetSchemaJdbcBudget( + JdbcMetadataMigrationDeadline.start(Duration.ofMillis(1500), ticker::get)); + Statement statement = mock(Statement.class); + + budget.apply(statement); + + verify(statement).setQueryTimeout(2); + ticker.set(Duration.ofSeconds(2).toNanos()); + assertThatThrownBy(() -> budget.apply(statement)) + .isInstanceOfSatisfying(MetadataMigrationException.class, failure -> + assertThat(failure.code()) + .isEqualTo(MetadataMigrationErrorCode.TIMEOUT)); + } + + @Test + void statementTimeoutConfigurationCannotConsumeTheRemainingBudget() throws Exception { + AtomicLong ticker = new AtomicLong(); + TargetSchemaJdbcBudget budget = new TargetSchemaJdbcBudget( + JdbcMetadataMigrationDeadline.start(Duration.ofNanos(1), ticker::get)); + Statement statement = mock(Statement.class); + doAnswer(invocation -> { + ticker.set(2); + return null; + }).when(statement).setQueryTimeout(1); + + assertThatThrownBy(() -> budget.apply(statement)) + .isInstanceOfSatisfying(MetadataMigrationException.class, failure -> + assertThat(failure.code()).isEqualTo(MetadataMigrationErrorCode.TIMEOUT)); + } + + @Test + void catalogRowsStopBeforeFieldReadsWhenCallerIsInterrupted() throws Exception { + Connection connection = mock(Connection.class); + PreparedStatement statement = mock(PreparedStatement.class); + ResultSet rows = mock(ResultSet.class); + when(connection.prepareStatement(anyString())).thenReturn(statement); + when(statement.executeQuery()).thenReturn(rows); + when(rows.next()).thenAnswer(invocation -> { + Thread.currentThread().interrupt(); + return true; + }); + TargetSchemaJdbcBudget budget = new TargetSchemaJdbcBudget( + JdbcMetadataMigrationDeadline.start(Duration.ofSeconds(5), System::nanoTime)); + + try { + assertThatThrownBy(() -> JdbcTargetSchemaObjectState.capture( + connection, + MetadataDatabaseKind.MYSQL, + Set.of("monitor"), + budget, + parts -> { })) + .isInstanceOfSatisfying(MetadataMigrationException.class, failure -> + assertThat(failure.code()) + .isEqualTo(MetadataMigrationErrorCode.TIMEOUT)); + verify(rows, never()).getString(1); + } finally { + Thread.interrupted(); + } + } + + @Test + void identityRowsUseTheSameCooperativeBudget() throws Exception { + Connection connection = mock(Connection.class); + PreparedStatement statement = mock(PreparedStatement.class); + ResultSet rows = interruptingRow(); + when(connection.prepareStatement(anyString())).thenReturn(statement); + when(statement.executeQuery()).thenReturn(rows); + TargetSchemaJdbcBudget budget = budget(); + + try { + assertThatThrownBy(() -> JdbcTargetSchemaObjectState.captureIdentityOwnership( + connection, + MetadataDatabaseKind.POSTGRESQL, + "monitor", + List.of("id"), + budget, + parts -> { })) + .isInstanceOf(MetadataMigrationException.class); + verify(rows, never()).getString(1); + } finally { + Thread.interrupted(); + } + } + + @Test + void recordedContractRowsUseTheSameCooperativeBudget() throws Exception { + Connection connection = mock(Connection.class); + PreparedStatement checks = mock(PreparedStatement.class); + PreparedStatement triggers = mock(PreparedStatement.class); + PreparedStatement recorded = mock(PreparedStatement.class); + ResultSet emptyChecks = mock(ResultSet.class); + ResultSet emptyTriggers = mock(ResultSet.class); + ResultSet recordedRows = interruptingRow(); + when(connection.prepareStatement(anyString())).thenReturn(checks, triggers, recorded); + when(checks.executeQuery()).thenReturn(emptyChecks); + when(triggers.executeQuery()).thenReturn(emptyTriggers); + when(recorded.executeQuery()).thenReturn(recordedRows); + + try { + assertThatThrownBy(() -> new TargetSchemaContract(MetadataDatabaseKind.MYSQL) + .matches(connection, Set.of(), budget())) + .isInstanceOf(MetadataMigrationException.class); + verify(recordedRows, never()).getString("database_kind"); + } finally { + Thread.interrupted(); + } + } + + private static TargetSchemaJdbcBudget budget() { + return new TargetSchemaJdbcBudget( + JdbcMetadataMigrationDeadline.start(Duration.ofSeconds(5), System::nanoTime)); + } + + private static ResultSet interruptingRow() throws Exception { + ResultSet rows = mock(ResultSet.class); + when(rows.next()).thenAnswer(invocation -> { + Thread.currentThread().interrupt(); + return true; + }); + return rows; + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaJdbcMetadataBudgetTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaJdbcMetadataBudgetTest.java new file mode 100644 index 0000000000..d76b2d1265 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaJdbcMetadataBudgetTest.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.time.Duration; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.api.Test; + +class TargetSchemaJdbcMetadataBudgetTest { + + @Test + void historyStopsAfterMetadataLookupBeforeCatalogLookup() throws Exception { + AtomicLong ticker = new AtomicLong(); + Connection connection = mock(Connection.class); + DatabaseMetaData metadata = mock(DatabaseMetaData.class); + when(connection.getMetaData()).thenAnswer(invocation -> { + ticker.set(2); + return metadata; + }); + + assertTimeout(() -> new FlywaySchemaHistory(MetadataDatabaseKind.MYSQL) + .requireEmptyTarget(connection, budget(ticker))); + verify(connection, never()).getCatalog(); + } + + @Test + void historyStopsAfterCatalogLookupBeforeTableLookup() throws Exception { + AtomicLong ticker = new AtomicLong(); + Connection connection = mock(Connection.class); + DatabaseMetaData metadata = mock(DatabaseMetaData.class); + ResultSet tables = mock(ResultSet.class); + when(connection.getMetaData()).thenReturn(metadata); + when(connection.getCatalog()).thenAnswer(invocation -> { + ticker.set(2); + return null; + }); + when(metadata.getTables(any(), any(), anyString(), any())).thenReturn(tables); + + assertTimeout(() -> new FlywaySchemaHistory(MetadataDatabaseKind.MYSQL) + .requireEmptyTarget(connection, budget(ticker))); + verify(metadata, never()).getTables(any(), any(), anyString(), any()); + } + + @Test + void historyStopsAfterTableLookupBeforeReadingRows() throws Exception { + AtomicLong ticker = new AtomicLong(); + Connection connection = mock(Connection.class); + DatabaseMetaData metadata = mock(DatabaseMetaData.class); + ResultSet tables = mock(ResultSet.class); + when(connection.getMetaData()).thenReturn(metadata); + when(metadata.getTables(any(), any(), anyString(), any())).thenAnswer(invocation -> { + ticker.set(2); + return tables; + }); + + assertTimeout(() -> new FlywaySchemaHistory(MetadataDatabaseKind.MYSQL) + .requireEmptyTarget(connection, budget(ticker))); + verify(tables, never()).next(); + verify(tables).close(); + } + + @Test + void semanticStateStopsAfterMetadataLookupBeforeCatalogLookup() throws Exception { + AtomicLong ticker = new AtomicLong(); + Connection connection = mock(Connection.class); + DatabaseMetaData metadata = mock(DatabaseMetaData.class); + when(connection.getMetaData()).thenAnswer(invocation -> { + ticker.set(2); + return metadata; + }); + + assertTimeout(() -> JdbcTargetSchemaState.capture( + connection, MetadataDatabaseKind.POSTGRESQL, Set.of(), budget(ticker))); + verify(connection, never()).getCatalog(); + } + + @Test + void semanticStateStopsAfterCatalogLookupBeforeSchemaLookup() throws Exception { + AtomicLong ticker = new AtomicLong(); + Connection connection = mock(Connection.class); + DatabaseMetaData metadata = mock(DatabaseMetaData.class); + when(connection.getMetaData()).thenReturn(metadata); + when(connection.getCatalog()).thenAnswer(invocation -> { + ticker.set(2); + return null; + }); + + assertTimeout(() -> JdbcTargetSchemaState.capture( + connection, MetadataDatabaseKind.POSTGRESQL, Set.of(), budget(ticker))); + verify(connection, never()).getSchema(); + } + + private static TargetSchemaJdbcBudget budget(AtomicLong ticker) { + return new TargetSchemaJdbcBudget( + JdbcMetadataMigrationDeadline.start(Duration.ofNanos(1), ticker::get)); + } + + private static void assertTimeout(ThrowingAction action) { + assertThatThrownBy(action::run) + .isInstanceOfSatisfying(MetadataMigrationException.class, failure -> + assertThat(failure.code()).isEqualTo(MetadataMigrationErrorCode.TIMEOUT)); + } + + @FunctionalInterface + private interface ThrowingAction { + + void run() throws Exception; + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisionerDatabaseTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisionerDatabaseTest.java index 271364de5b..30c769bfad 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisionerDatabaseTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaProvisionerDatabaseTest.java @@ -29,6 +29,7 @@ import java.sql.DatabaseMetaData; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.Statement; +import java.time.Duration; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -200,6 +201,7 @@ class TargetSchemaProvisionerDatabaseTest { MetadataDatabaseConfiguration target = new MetadataDatabaseConfiguration(kind, jdbcUrl, USERNAME, PASSWORD); TargetSchemaProvisioner provisioner = new FlywayTargetSchemaProvisioner(); + assertCallerOwnedProvisioningIsCurrentAndReusable(target); assertProvisioningLogsAreSanitized(provisioner, target); assertCurrentBaselineAllowsAdditionalTable(provisioner, target); assertCurrentBaselineRejectsSchemaCorruption(provisioner, target); @@ -210,10 +212,12 @@ class TargetSchemaProvisionerDatabaseTest { TargetSchemaBaselineResourceTest.mappedTables()); try (Statement statement = connection.createStatement(); ResultSet history = statement.executeQuery( - "SELECT version, type, success FROM flyway_schema_history ORDER BY installed_rank")) { + "SELECT version, type, installed_by, success " + + "FROM flyway_schema_history ORDER BY installed_rank")) { assertThat(history.next()).isTrue(); assertThat(history.getString("version")).isEqualTo("206"); assertThat(history.getString("type")).isEqualTo("SQL_BASELINE"); + assertThat(history.getString("installed_by")).isEqualTo("hertzbeat-migration"); assertThat(history.getBoolean("success")).isTrue(); assertThat(history.next()).isFalse(); } @@ -241,6 +245,29 @@ class TargetSchemaProvisionerDatabaseTest { } } + private static void assertCallerOwnedProvisioningIsCurrentAndReusable( + MetadataDatabaseConfiguration target) throws Exception { + FlywayTargetSchemaProvisioner provisioner = new FlywayTargetSchemaProvisioner(); + try (Connection connection = DriverManager.getConnection( + target.jdbcUrl(), target.username(), target.password())) { + JdbcMetadataMigrationDeadline firstDeadline = JdbcMetadataMigrationDeadline.start( + Duration.ofMinutes(2), System::nanoTime); + TargetSchemaProvisioningOutcome first = provisioner.provision( + connection, target.kind(), firstDeadline); + assertThat(first.disposition()).isEqualTo(TargetSchemaConnectionDisposition.REUSABLE); + assertThat(connection.isClosed()).isFalse(); + assertThat(connection.getAutoCommit()).isTrue(); + + JdbcMetadataMigrationDeadline currentDeadline = JdbcMetadataMigrationDeadline.start( + Duration.ofMinutes(2), System::nanoTime); + TargetSchemaProvisioningOutcome current = provisioner.provision( + connection, target.kind(), currentDeadline); + assertThat(current.disposition()).isEqualTo(TargetSchemaConnectionDisposition.REUSABLE); + assertThat(connection.isClosed()).isFalse(); + assertThat(connection.getAutoCommit()).isTrue(); + } + } + private static List schemaDifferences( MetadataSchemaSnapshot baseline, MetadataSchemaSnapshot migrated) { List differences = new ArrayList<>(); From 8efbb0260398f1319e03bff81e21df3b03374e58 Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 11:53:17 +0800 Subject: [PATCH 47/71] Bound target JDBC connections --- hertzbeat-manager/pom.xml | 5 + .../setup/workflow/TargetJdbcCleanupLane.java | 239 ++++++++ .../setup/workflow/TargetJdbcCleanupTask.java | 116 ++++ .../workflow/TargetJdbcCleanupWorker.java | 153 +++++ .../workflow/TargetJdbcConnectionAction.java | 17 + .../workflow/TargetJdbcConnectionAttempt.java | 220 +++++++ .../TargetJdbcConnectionAttemptOwner.java | 22 + .../TargetJdbcConnectionErrorCode.java | 18 + .../TargetJdbcConnectionException.java | 23 + .../workflow/TargetJdbcConnectionFactory.java | 176 ++++++ .../workflow/TargetJdbcConnectionLease.java | 137 +++++ .../TargetJdbcConnectionVerifier.java | 164 ++++++ .../setup/workflow/TargetJdbcConnector.java | 22 + .../TargetJdbcDataSourceProvider.java | 18 + .../TargetJdbcDataSourceSettings.java | 33 ++ .../setup/workflow/TargetJdbcEndpoint.java | 89 +++ .../setup/workflow/TargetJdbcIdentity.java | 59 ++ .../workflow/TargetJdbcResultWaiter.java | 21 + .../manager/setup/workflow/TargetJdbcUrl.java | 263 +++++++++ .../workflow/TargetJdbcVendorConnector.java | 89 +++ .../workflow/TargetJdbcCleanupLaneTest.java | 512 +++++++++++++++++ ...getJdbcConnectionFactoryLifecycleTest.java | 349 ++++++++++++ .../TargetJdbcConnectionFactoryTest.java | 535 ++++++++++++++++++ .../TargetJdbcConnectionLeaseTest.java | 197 +++++++ .../TargetJdbcConnectionVerifierTest.java | 304 ++++++++++ .../workflow/TargetJdbcIdentityTest.java | 56 ++ .../setup/workflow/TargetJdbcUrlTest.java | 149 +++++ .../TargetJdbcVendorConnectorTest.java | 141 +++++ ...rgetJdbcConnectionFactoryDatabaseTest.java | 148 +++++ 29 files changed, 4275 insertions(+) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcCleanupLane.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcCleanupTask.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcCleanupWorker.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionAction.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionAttempt.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionAttemptOwner.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionErrorCode.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionException.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionFactory.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionLease.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionVerifier.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnector.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcDataSourceProvider.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcDataSourceSettings.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcEndpoint.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcIdentity.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcResultWaiter.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcUrl.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcVendorConnector.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcCleanupLaneTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionFactoryLifecycleTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionFactoryTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionLeaseTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionVerifierTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcIdentityTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcUrlTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcVendorConnectorTest.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionFactoryDatabaseTest.java diff --git a/hertzbeat-manager/pom.xml b/hertzbeat-manager/pom.xml index 8d8a40349f..82ac33eb7b 100644 --- a/hertzbeat-manager/pom.xml +++ b/hertzbeat-manager/pom.xml @@ -151,6 +151,11 @@ mysql-connector-j provided + + org.postgresql + postgresql + provided + org.springframework.boot diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcCleanupLane.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcCleanupLane.java new file mode 100644 index 0000000000..1b908b314d --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcCleanupLane.java @@ -0,0 +1,239 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.util.ArrayDeque; +import java.util.Collections; +import java.util.Deque; +import java.util.IdentityHashMap; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.Executor; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** Owns the exact provisional connection while bounded cleanup is pending. */ +final class TargetJdbcCleanupLane implements AutoCloseable { + + private final TargetJdbcCleanupWorker worker; + private final Executor abortExecutor; + private final Deque retained = new ArrayDeque<>(); + private final Set retainedIdentities = Collections.newSetFromMap(new IdentityHashMap<>()); + private TargetJdbcCleanupTask active; + private Error pendingFatal; + private boolean poisoned; + private boolean closing; + + TargetJdbcCleanupLane(ThreadPoolExecutor worker, Executor abortExecutor) { + this.worker = new TargetJdbcCleanupWorker(worker); + this.abortExecutor = Objects.requireNonNull(abortExecutor, "abortExecutor"); + } + + synchronized TargetJdbcConnectionErrorCode acquisitionFailure() { + if (!retained.isEmpty() || active != null || pendingFatal != null) { + return TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED; + } + return poisoned ? TargetJdbcConnectionErrorCode.FACTORY_CLOSED : null; + } + + synchronized void poison(Connection connection) { + poisoned = true; + retainLocked(connection); + } + + void cleanupLate(Connection connection) { + TargetJdbcCleanupTask task; + synchronized (this) { + poisoned = true; + retainLocked(connection); + if (active != null || pendingFatal != null) { + return; + } + task = newTask(firstRetainedLocked(), true); + active = task; + } + submit(task); + } + + void retry(JdbcMetadataMigrationDeadline deadline) { + boolean attempted = false; + while (true) { + replayPendingFatal(); + TargetJdbcCleanupTask task = claimRetry(attempted, deadline); + if (task == null) { + return; + } + attempted = true; + await(task, deadline); + try { + replay(task); + } catch (TargetJdbcConnectionException cleanupRequired) { + if (!task.abortFirst()) { + throw cleanupRequired; + } + continue; + } + } + } + + private TargetJdbcCleanupTask claimRetry( + boolean attempted, + JdbcMetadataMigrationDeadline deadline) { + TargetJdbcCleanupTask task; + synchronized (this) { + if (retained.isEmpty()) { + if (!attempted) { + throw failure(TargetJdbcConnectionErrorCode.FACTORY_CLOSED); + } + return null; + } + if (active != null) { + return active; + } + task = newTask(firstRetainedLocked(), false); + active = task; + } + submitRetry(task, deadline); + return task; + } + + private void submitRetry( + TargetJdbcCleanupTask task, + JdbcMetadataMigrationDeadline deadline) { + worker.submitWhenAvailable(task, deadline, this::submissionFailed); + } + + private static void await(TargetJdbcCleanupTask task, JdbcMetadataMigrationDeadline deadline) { + boolean interrupted = false; + try { + long remaining = deadline.remainingNanos(); + if (remaining <= 0 || !task.completed().await(remaining, TimeUnit.NANOSECONDS)) { + throw failure(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED); + } + } catch (InterruptedException interruptedFailure) { + interrupted = true; + throw failure(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED); + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + + @Override + public void close() { + TargetJdbcCleanupTask task = null; + synchronized (this) { + closing = true; + if (active == null && pendingFatal == null && !retained.isEmpty()) { + task = newTask(firstRetainedLocked(), false); + active = task; + } else if (active == null && retained.isEmpty()) { + worker.shutdown(); + } + } + if (task != null) { + worker.submitWhenAvailableAsync(task, this::submissionFailed); + } + } + + private void submit(TargetJdbcCleanupTask task) { + worker.submit(task, this::submissionFailed); + } + + private void submissionFailed(TargetJdbcCleanupTask task, Error fatal) { + submitNext(finish(task, false, fatal)); + } + + private TargetJdbcCleanupTask finish(TargetJdbcCleanupTask task, boolean success, Error fatal) { + TargetJdbcCleanupTask next = null; + boolean shutdown = false; + synchronized (this) { + if (active != task) { + return null; + } + if (success) { + success = removeFirstRetainedLocked(task.connection()); + if (!success) { + poisoned = true; + } + } else { + poisoned = true; + } + if (fatal != null && pendingFatal == null) { + pendingFatal = fatal; + } + active = null; + task.complete(success, fatal); + if (closing && success && pendingFatal == null && !retained.isEmpty()) { + next = newTask(firstRetainedLocked(), false); + active = next; + } else if (closing && retained.isEmpty()) { + shutdown = true; + } + } + if (shutdown) { + worker.shutdown(); + } + return next; + } + + private void submitNext(TargetJdbcCleanupTask next) { + if (next != null) { + submit(next); + } + } + + private synchronized void replayPendingFatal() { + if (pendingFatal != null) { + Error fatal = pendingFatal; + pendingFatal = null; + throw fatal; + } + } + + private void replay(TargetJdbcCleanupTask task) { + if (task.fatal() != null) { + synchronized (this) { + if (pendingFatal == task.fatal()) { + pendingFatal = null; + } + } + } + task.replay(); + } + + private void retainLocked(Connection connection) { + if (connection == null || !retainedIdentities.add(connection)) { + return; + } + retained.addLast(connection); + } + + private Connection firstRetainedLocked() { + return retained.getFirst(); + } + + private boolean removeFirstRetainedLocked(Connection connection) { + if (retained.peekFirst() != connection || !retainedIdentities.contains(connection)) { + return false; + } + retained.removeFirst(); + return retainedIdentities.remove(connection); + } + + private TargetJdbcCleanupTask newTask(Connection connection, boolean abortFirst) { + return new TargetJdbcCleanupTask(connection, abortFirst, abortExecutor, this::finish); + } + + private static TargetJdbcConnectionException failure(TargetJdbcConnectionErrorCode code) { + return new TargetJdbcConnectionException(code); + } + +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcCleanupTask.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcCleanupTask.java new file mode 100644 index 0000000000..a2e9f8f0e6 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcCleanupTask.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Objects; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; + +/** Executes one exact provisional connection cleanup without leaking driver diagnostics. */ +final class TargetJdbcCleanupTask implements Runnable { + + private final Connection connection; + private final boolean abortFirst; + private final Executor abortExecutor; + private final Completion completion; + private final CountDownLatch completed = new CountDownLatch(1); + private boolean success; + private Error fatal; + + TargetJdbcCleanupTask( + Connection connection, + boolean abortFirst, + Executor abortExecutor, + Completion completion) { + this.connection = Objects.requireNonNull(connection, "connection"); + this.abortFirst = abortFirst; + this.abortExecutor = Objects.requireNonNull(abortExecutor, "abortExecutor"); + this.completion = Objects.requireNonNull(completion, "completion"); + } + + @Override + public void run() { + TargetJdbcCleanupTask task = this; + while (task != null) { + task = task.runOnce(); + } + } + + private TargetJdbcCleanupTask runOnce() { + boolean interrupted = Thread.interrupted(); + Error cleanupFatal = null; + boolean closed = false; + TargetJdbcCleanupTask next = null; + try { + if (abortFirst) { + try { + connection.abort(abortExecutor); + } catch (SQLException | RuntimeException ignored) { + // Exact close remains mandatory after an abort failure. + } finally { + interrupted |= Thread.interrupted(); + } + } + connection.close(); + closed = true; + } catch (SQLException | RuntimeException cleanupFailure) { + // Stable cleanup-required state contains no driver diagnostic. + } catch (Error fatalCleanup) { + cleanupFatal = fatalCleanup; + } finally { + interrupted |= Thread.interrupted(); + try { + next = completion.finished(this, closed, cleanupFatal); + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + return next; + } + + Connection connection() { + return connection; + } + + boolean abortFirst() { + return abortFirst; + } + + CountDownLatch completed() { + return completed; + } + + Error fatal() { + return fatal; + } + + void complete(boolean completedSuccessfully, Error completedFatal) { + success = completedSuccessfully; + fatal = completedFatal; + completed.countDown(); + } + + void replay() { + if (fatal != null) { + throw fatal; + } + if (!success) { + throw new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED); + } + } + + @FunctionalInterface + interface Completion { + + TargetJdbcCleanupTask finished(TargetJdbcCleanupTask task, boolean success, Error fatal); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcCleanupWorker.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcCleanupWorker.java new file mode 100644 index 0000000000..e4d075c420 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcCleanupWorker.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.LockSupport; + +/** Submits exact cleanup work without turning a zero-queue worker handoff into a cleanup failure. */ +final class TargetJdbcCleanupWorker { + + private static final long AVAILABILITY_POLL_NANOS = TimeUnit.MILLISECONDS.toNanos(1); + + private final ThreadPoolExecutor executor; + + TargetJdbcCleanupWorker(ThreadPoolExecutor executor) { + this.executor = Objects.requireNonNull(executor, "executor"); + } + + void submit(TargetJdbcCleanupTask task, SubmissionFailure failure) { + try { + executor.execute(task); + } catch (RuntimeException submissionFailure) { + failure.failed(task, null); + } catch (Error fatalSubmission) { + failure.failed(task, fatalSubmission); + } + } + + void submitWhenAvailableAsync(TargetJdbcCleanupTask task, SubmissionFailure failure) { + try { + executor.execute(task); + } catch (RejectedExecutionException transientRejection) { + if (executor.isShutdown()) { + failure.failed(task, null); + return; + } + startHandoff(task, failure); + } catch (RuntimeException submissionFailure) { + failure.failed(task, null); + } catch (Error fatalSubmission) { + failure.failed(task, fatalSubmission); + } + } + + void submitWhenAvailable( + TargetJdbcCleanupTask task, + JdbcMetadataMigrationDeadline deadline, + SubmissionFailure failure) { + boolean interrupted = false; + try { + while (true) { + if (deadline.remainingNanos() <= 0) { + failure.failed(task, null); + return; + } + try { + executor.execute(task); + return; + } catch (RejectedExecutionException transientRejection) { + if (executor.isShutdown()) { + failure.failed(task, null); + return; + } + long remaining = deadline.remainingNanos(); + if (remaining <= 0) { + failure.failed(task, null); + return; + } + LockSupport.parkNanos(Math.min(remaining, AVAILABILITY_POLL_NANOS)); + if (Thread.interrupted()) { + interrupted = true; + failure.failed(task, null); + return; + } + } catch (RuntimeException submissionFailure) { + failure.failed(task, null); + return; + } catch (Error fatalSubmission) { + failure.failed(task, fatalSubmission); + return; + } + } + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + + void shutdown() { + executor.shutdown(); + } + + private void startHandoff(TargetJdbcCleanupTask task, SubmissionFailure failure) { + try { + Thread.ofPlatform() + .daemon(true) + .name("target-jdbc-cleanup-close-handoff") + .start(() -> awaitAvailability(task, failure)); + } catch (RuntimeException submissionFailure) { + failure.failed(task, null); + } catch (Error fatalSubmission) { + failure.failed(task, fatalSubmission); + } + } + + private void awaitAvailability(TargetJdbcCleanupTask task, SubmissionFailure failure) { + boolean interrupted = false; + try { + while (true) { + try { + executor.execute(task); + return; + } catch (RejectedExecutionException transientRejection) { + if (executor.isShutdown()) { + failure.failed(task, null); + return; + } + LockSupport.parkNanos(AVAILABILITY_POLL_NANOS); + if (Thread.interrupted()) { + interrupted = true; + failure.failed(task, null); + return; + } + } catch (RuntimeException submissionFailure) { + failure.failed(task, null); + return; + } catch (Error fatalSubmission) { + failure.failed(task, fatalSubmission); + return; + } + } + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + + @FunctionalInterface + interface SubmissionFailure { + + void failed(TargetJdbcCleanupTask task, Error fatal); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionAction.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionAction.java new file mode 100644 index 0000000000..983d5ebc56 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionAction.java @@ -0,0 +1,17 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; + +/** Synchronous action that must not retain, replace, or close the leased connection. */ +@FunctionalInterface +interface TargetJdbcConnectionAction { + + void execute(Connection connection); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionAttempt.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionAttempt.java new file mode 100644 index 0000000000..5a3bb1f02a --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionAttempt.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Objects; +import java.util.concurrent.CountDownLatch; + +/** One deadline-bound connection attempt and its publication race. */ +final class TargetJdbcConnectionAttempt implements Runnable { + + private final TargetJdbcConnectionAttemptOwner owner; + private final TargetJdbcConnector connector; + private final TargetJdbcConnectionVerifier verifier; + private final TargetJdbcUrl target; + private final String username; + private final char[] password; + private final JdbcMetadataMigrationDeadline deadline; + private final TargetJdbcResultWaiter resultWaiter; + private final CountDownLatch resultReady = new CountDownLatch(1); + private State state = State.RUNNING; + private TargetJdbcConnectionLease lease; + private TargetJdbcConnectionException failure; + private Error fatal; + + TargetJdbcConnectionAttempt( + TargetJdbcConnectionAttemptOwner owner, + TargetJdbcConnector connector, + TargetJdbcConnectionVerifier verifier, + TargetJdbcUrl target, + String username, + char[] password, + JdbcMetadataMigrationDeadline deadline, + TargetJdbcResultWaiter resultWaiter) { + this.owner = Objects.requireNonNull(owner, "owner"); + this.connector = Objects.requireNonNull(connector, "connector"); + this.verifier = Objects.requireNonNull(verifier, "verifier"); + this.target = Objects.requireNonNull(target, "target"); + this.username = Objects.requireNonNull(username, "username"); + this.password = Objects.requireNonNull(password, "password"); + this.deadline = Objects.requireNonNull(deadline, "deadline"); + this.resultWaiter = Objects.requireNonNull(resultWaiter, "resultWaiter"); + } + + @Override + public void run() { + Connection connection = null; + try { + connection = connector.connect(target, username, password, deadline); + if (abandoned()) { + closeLate(connection); + return; + } + TargetJdbcConnectionLease verified = verifier.verify(connection, target, username, deadline); + if (!complete(verified, null, null)) { + closeLate(connection); + } + } catch (TargetJdbcConnectionException connectionFailure) { + if (connectionFailure.code() == TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED + && connection != null) { + owner.poison(connection); + } + complete(null, connectionFailure, null); + } catch (MetadataMigrationException deadlineFailure) { + TargetJdbcConnectionErrorCode code = deadlineFailure.code() == MetadataMigrationErrorCode.TIMEOUT + ? TargetJdbcConnectionErrorCode.TIMEOUT + : TargetJdbcConnectionErrorCode.UNAVAILABLE; + complete(null, failure(code), null); + } catch (SQLException | RuntimeException unavailable) { + complete(null, failure(TargetJdbcConnectionErrorCode.UNAVAILABLE), null); + } catch (Error error) { + if (complete(null, null, error)) { + owner.poison(connection); + } else { + owner.lateFatal(error, connection); + } + } finally { + Arrays.fill(password, '\0'); + try { + owner.finished(this); + } catch (RuntimeException lifecycleFailure) { + recordLifecycleFailure(lifecycleFailure, connection); + } catch (Error fatalLifecycle) { + recordLifecycleFailure(fatalLifecycle, connection); + } finally { + publishResult(); + } + } + } + + TargetJdbcConnectionLease await() { + boolean interrupted = false; + try { + long remaining = deadline.remainingNanos(); + if (remaining <= 0 || !resultWaiter.await(resultReady, remaining)) { + if (abandon(TargetJdbcConnectionErrorCode.TIMEOUT)) { + throw failure(TargetJdbcConnectionErrorCode.TIMEOUT); + } + awaitPublication(); + return replay(); + } + } catch (InterruptedException interruptedFailure) { + interrupted = true; + if (abandon(TargetJdbcConnectionErrorCode.TIMEOUT)) { + throw failure(TargetJdbcConnectionErrorCode.TIMEOUT); + } + awaitPublication(); + return replay(); + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + return replay(); + } + + synchronized boolean abandon(TargetJdbcConnectionErrorCode code) { + if (state == State.RUNNING) { + failure = failure(code); + state = State.ABANDONED; + resultReady.countDown(); + return true; + } + return false; + } + + private synchronized boolean abandoned() { + return state == State.ABANDONED; + } + + private synchronized boolean complete( + TargetJdbcConnectionLease completedLease, + TargetJdbcConnectionException completedFailure, + Error completedFatal) { + if (state != State.RUNNING) { + return false; + } + lease = completedLease; + failure = completedFailure; + fatal = completedFatal; + state = State.RESULT_READY; + return true; + } + + private void recordLifecycleFailure(Throwable lifecycleFailure, Connection connection) { + boolean accepted; + synchronized (this) { + accepted = state == State.RESULT_READY; + if (accepted) { + lease = null; + if (lifecycleFailure instanceof Error lifecycleFatal) { + fatal = lifecycleFatal; + failure = null; + } else { + failure = failure(TargetJdbcConnectionErrorCode.UNAVAILABLE); + fatal = null; + } + } + } + if (accepted) { + owner.poison(connection); + } else if (lifecycleFailure instanceof Error lifecycleFatal) { + owner.lateFatal(lifecycleFatal, connection); + } else { + owner.poison(connection); + } + } + + private synchronized void publishResult() { + if (state == State.RESULT_READY) { + resultReady.countDown(); + } + } + + private void awaitPublication() { + boolean interrupted = false; + while (true) { + try { + resultReady.await(); + break; + } catch (InterruptedException ignored) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + private synchronized TargetJdbcConnectionLease replay() { + if (fatal != null) { + throw fatal; + } + if (failure != null) { + throw failure; + } + return lease; + } + + private void closeLate(Connection connection) { + owner.cleanupLate(connection); + } + + private static TargetJdbcConnectionException failure(TargetJdbcConnectionErrorCode code) { + return new TargetJdbcConnectionException(code); + } + + private enum State { + RUNNING, + ABANDONED, + RESULT_READY + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionAttemptOwner.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionAttemptOwner.java new file mode 100644 index 0000000000..7a5fb2c6be --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionAttemptOwner.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; + +/** Lifecycle callbacks from one target JDBC acquisition attempt. */ +interface TargetJdbcConnectionAttemptOwner { + + void poison(Connection connection); + + void cleanupLate(Connection connection); + + void lateFatal(Error failure, Connection connection); + + void finished(TargetJdbcConnectionAttempt attempt); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionErrorCode.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionErrorCode.java new file mode 100644 index 0000000000..83f584ad2d --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionErrorCode.java @@ -0,0 +1,18 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +/** Stable, credential-free failures for target JDBC connection ownership. */ +enum TargetJdbcConnectionErrorCode { + TIMEOUT, + UNAVAILABLE, + TARGET_MISMATCH, + OPERATION_CONFLICT, + FACTORY_CLOSED, + CLEANUP_REQUIRED +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionException.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionException.java new file mode 100644 index 0000000000..0208980395 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionException.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +/** Cause-free target connection failure safe for workflow and API translation. */ +final class TargetJdbcConnectionException extends RuntimeException { + + private final TargetJdbcConnectionErrorCode code; + + TargetJdbcConnectionException(TargetJdbcConnectionErrorCode code) { + super("Target JDBC connection failed: " + code.name().toLowerCase(java.util.Locale.ROOT)); + this.code = java.util.Objects.requireNonNull(code, "code"); + } + + TargetJdbcConnectionErrorCode code() { + return code; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionFactory.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionFactory.java new file mode 100644 index 0000000000..8b09c94cc0 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionFactory.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.util.Arrays; +import java.util.Objects; +import java.util.concurrent.Executor; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; + +/** Owns one bounded target JDBC acquisition worker and any failed cleanup handle. */ +final class TargetJdbcConnectionFactory implements AutoCloseable, TargetJdbcConnectionAttemptOwner { + + private final ThreadPoolExecutor worker; + private final TargetJdbcCleanupLane cleanupLane; + private final TargetJdbcConnector connector; + private final TargetJdbcConnectionVerifier verifier; + private final TargetJdbcResultWaiter resultWaiter; + private TargetJdbcConnectionAttempt active; + private Error lateFailure; + private boolean closed; + + TargetJdbcConnectionFactory(Executor abortExecutor) { + this(new TargetJdbcVendorConnector(), abortExecutor); + } + + TargetJdbcConnectionFactory(TargetJdbcConnector connector, Executor abortExecutor) { + this(newWorker("target-jdbc-acquisition"), newWorker("target-jdbc-cleanup"), + abortExecutor, connector, new TargetJdbcConnectionVerifier(abortExecutor)); + } + + TargetJdbcConnectionFactory( + ThreadPoolExecutor worker, + ThreadPoolExecutor cleanupWorker, + Executor abortExecutor, + TargetJdbcConnector connector, + TargetJdbcConnectionVerifier verifier) { + this(worker, cleanupWorker, abortExecutor, connector, verifier, TargetJdbcResultWaiter.TIMED); + } + + TargetJdbcConnectionFactory( + ThreadPoolExecutor worker, + ThreadPoolExecutor cleanupWorker, + Executor abortExecutor, + TargetJdbcConnector connector, + TargetJdbcConnectionVerifier verifier, + TargetJdbcResultWaiter resultWaiter) { + this.worker = Objects.requireNonNull(worker, "worker"); + this.cleanupLane = new TargetJdbcCleanupLane(cleanupWorker, abortExecutor); + this.connector = Objects.requireNonNull(connector, "connector"); + this.verifier = Objects.requireNonNull(verifier, "verifier"); + this.resultWaiter = Objects.requireNonNull(resultWaiter, "resultWaiter"); + } + + TargetJdbcConnectionLease acquire( + MetadataDatabaseSettings settings, + SecretValue borrowedPassword, + JdbcMetadataMigrationDeadline deadline) { + Objects.requireNonNull(settings, "settings"); + Objects.requireNonNull(borrowedPassword, "borrowedPassword"); + Objects.requireNonNull(deadline, "deadline"); + TargetJdbcUrl target = TargetJdbcUrl.parse(settings.kind(), settings.jdbcUrl()); + char[] attemptPassword = borrowedPassword.copy(); + TargetJdbcConnectionAttempt attempt = new TargetJdbcConnectionAttempt( + this, connector, verifier, target, settings.username(), + attemptPassword, deadline, resultWaiter); + try { + claim(attempt); + } catch (RuntimeException | Error claimFailure) { + Arrays.fill(attemptPassword, '\0'); + throw claimFailure; + } + try { + worker.execute(attempt); + } catch (RejectedExecutionException rejected) { + Arrays.fill(attemptPassword, '\0'); + poison(null); + finished(attempt); + throw failure(TargetJdbcConnectionErrorCode.FACTORY_CLOSED); + } catch (RuntimeException submitFailure) { + Arrays.fill(attemptPassword, '\0'); + poison(null); + finished(attempt); + throw failure(TargetJdbcConnectionErrorCode.FACTORY_CLOSED); + } catch (Error fatalSubmission) { + Arrays.fill(attemptPassword, '\0'); + lateFatal(fatalSubmission, null); + finished(attempt); + throw fatalSubmission; + } + return attempt.await(); + } + + void retryCleanup(JdbcMetadataMigrationDeadline deadline) { + cleanupLane.retry(deadline); + } + + @Override + public synchronized void close() { + closed = true; + if (active != null) { + active.abandon(TargetJdbcConnectionErrorCode.FACTORY_CLOSED); + } else { + cleanupLane.close(); + } + worker.shutdownNow(); + } + + @Override + public synchronized void poison(Connection connection) { + cleanupLane.poison(connection); + } + + @Override + public void cleanupLate(Connection connection) { + cleanupLane.cleanupLate(connection); + } + + @Override + public synchronized void lateFatal(Error failure, Connection connection) { + cleanupLane.poison(connection); + if (lateFailure == null) { + lateFailure = failure; + } + } + + @Override + public synchronized void finished(TargetJdbcConnectionAttempt attempt) { + if (active == attempt) { + active = null; + if (closed) { + cleanupLane.close(); + } + } + } + + private synchronized void claim(TargetJdbcConnectionAttempt attempt) { + if (lateFailure != null) { + throw lateFailure; + } + TargetJdbcConnectionErrorCode cleanupFailure = cleanupLane.acquisitionFailure(); + if (cleanupFailure != null) { + throw failure(cleanupFailure); + } + if (closed) { + throw failure(TargetJdbcConnectionErrorCode.FACTORY_CLOSED); + } + if (active != null) { + throw failure(TargetJdbcConnectionErrorCode.OPERATION_CONFLICT); + } + active = attempt; + } + + private static TargetJdbcConnectionException failure(TargetJdbcConnectionErrorCode code) { + return new TargetJdbcConnectionException(code); + } + + private static ThreadPoolExecutor newWorker(String name) { + ThreadPoolExecutor executor = new ThreadPoolExecutor( + 0, 1, 30, TimeUnit.SECONDS, new SynchronousQueue<>(), + Thread.ofPlatform().daemon(true).name(name, 0).factory(), + new ThreadPoolExecutor.AbortPolicy()); + executor.allowCoreThreadTimeOut(true); + return executor; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionLease.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionLease.java new file mode 100644 index 0000000000..6ecd37bcf1 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionLease.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Objects; + +/** Exclusive scoped ownership of one verified target JDBC connection. */ +final class TargetJdbcConnectionLease implements AutoCloseable { + + private final Connection connection; + private final String targetIdentityHash; + private boolean callbackActive; + private Thread callbackOwner; + private boolean closing; + private boolean closeInProgress; + private boolean closed; + + TargetJdbcConnectionLease(Connection connection, String targetIdentityHash) { + this.connection = Objects.requireNonNull(connection, "connection"); + if (targetIdentityHash == null || !targetIdentityHash.matches("[0-9a-f]{64}")) { + throw new IllegalArgumentException("Invalid target JDBC identity"); + } + this.targetIdentityHash = targetIdentityHash; + } + + String targetIdentityHash() { + return targetIdentityHash; + } + + void withConnection(TargetJdbcConnectionAction action) { + Objects.requireNonNull(action, "action"); + beginCallback(); + try { + action.execute(connection); + } finally { + endCallback(); + } + } + + @Override + public void close() { + boolean interrupted = Thread.interrupted(); + boolean claimed = false; + boolean success = false; + try { + CloseClaim claim = claimClose(); + interrupted |= claim.interrupted(); + if (!claim.execute()) { + return; + } + claimed = true; + interrupted |= Thread.interrupted(); + try { + connection.close(); + success = true; + } catch (SQLException | RuntimeException cleanupFailure) { + throw new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED); + } + } finally { + interrupted |= Thread.interrupted(); + if (claimed) { + completeClose(success); + } + restoreInterrupt(interrupted); + } + } + + @Override + public String toString() { + return "TargetJdbcConnectionLease[targetIdentityHash=" + targetIdentityHash + ']'; + } + + private synchronized void beginCallback() { + if (closed || closing || callbackActive) { + throw conflict(); + } + callbackActive = true; + callbackOwner = Thread.currentThread(); + } + + private synchronized void endCallback() { + callbackActive = false; + callbackOwner = null; + notifyAll(); + } + + private synchronized CloseClaim claimClose() { + if (closed) { + return new CloseClaim(false, false); + } + if (callbackActive && callbackOwner == Thread.currentThread()) { + throw conflict(); + } + closing = true; + boolean interrupted = false; + while (callbackActive || closeInProgress) { + try { + wait(); + } catch (InterruptedException ignored) { + interrupted = true; + } + } + if (closed) { + return new CloseClaim(false, interrupted); + } + closeInProgress = true; + return new CloseClaim(true, interrupted); + } + + private synchronized void completeClose(boolean success) { + closeInProgress = false; + if (success) { + closed = true; + } + notifyAll(); + } + + private static void restoreInterrupt(boolean interrupted) { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + private static TargetJdbcConnectionException conflict() { + return new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.OPERATION_CONFLICT); + } + + private record CloseClaim(boolean execute, boolean interrupted) { + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionVerifier.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionVerifier.java new file mode 100644 index 0000000000..82f93949d2 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionVerifier.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.util.Objects; +import java.util.concurrent.Executor; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Verifies a provisional connection before transferring it into a scoped lease. */ +final class TargetJdbcConnectionVerifier { + + private final Executor networkExecutor; + + TargetJdbcConnectionVerifier(Executor networkExecutor) { + this.networkExecutor = Objects.requireNonNull(networkExecutor, "networkExecutor"); + } + + TargetJdbcConnectionLease verify( + Connection connection, + TargetJdbcUrl configured, + String username, + JdbcMetadataMigrationDeadline deadline) { + Objects.requireNonNull(connection, "connection"); + Objects.requireNonNull(configured, "configured"); + Objects.requireNonNull(deadline, "deadline"); + try { + require(call(deadline, connection::getAutoCommit)); + require(!call(deadline, connection::isReadOnly)); + int networkTimeout = remainingMillis(deadline); + run(deadline, () -> connection.setNetworkTimeout(networkExecutor, networkTimeout)); + DatabaseMetaData metadata = call(deadline, connection::getMetaData); + String product = call(deadline, metadata::getDatabaseProductName); + require(expectedProduct(configured.kind()).equals(product)); + String actualUrl = call(deadline, metadata::getURL); + TargetJdbcEndpoint actual = parseActual(configured.kind(), actualUrl); + require(actual.matches(configured)); + String catalog = call(deadline, connection::getCatalog); + require(configured.database().equals(catalog)); + String schema = configured.kind() == MetadataDatabaseKind.POSTGRESQL + ? call(deadline, connection::getSchema) : null; + if (configured.kind() == MetadataDatabaseKind.POSTGRESQL) { + require(schema != null && !schema.isBlank()); + } + check(deadline); + return new TargetJdbcConnectionLease( + connection, TargetJdbcIdentity.hash(configured, username, catalog, schema)); + } catch (TargetJdbcConnectionException failure) { + closeAfterFailure(connection, failure); + throw failure; + } catch (MetadataMigrationException timeout) { + TargetJdbcConnectionException failure = failure(TargetJdbcConnectionErrorCode.TIMEOUT); + closeAfterFailure(connection, failure); + throw failure; + } catch (SQLException | RuntimeException unavailable) { + TargetJdbcConnectionException failure = failure(TargetJdbcConnectionErrorCode.UNAVAILABLE); + closeAfterFailure(connection, failure); + throw failure; + } catch (Error fatal) { + closeAfterFatal(connection, fatal); + throw fatal; + } + } + + private static TargetJdbcEndpoint parseActual(MetadataDatabaseKind kind, String actualUrl) { + try { + return TargetJdbcEndpoint.parse(kind, actualUrl); + } catch (IllegalArgumentException invalid) { + throw failure(TargetJdbcConnectionErrorCode.TARGET_MISMATCH); + } + } + + private static T call(JdbcMetadataMigrationDeadline deadline, JdbcCall call) throws SQLException { + check(deadline); + T value = call.execute(); + check(deadline); + return value; + } + + private static void run(JdbcMetadataMigrationDeadline deadline, JdbcAction action) throws SQLException { + check(deadline); + action.execute(); + check(deadline); + } + + private static int remainingMillis(JdbcMetadataMigrationDeadline deadline) { + check(deadline); + return deadline.remainingMillis(); + } + + private static void check(JdbcMetadataMigrationDeadline deadline) { + if (Thread.currentThread().isInterrupted()) { + throw new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT); + } + deadline.remainingDuration(); + } + + private static void require(boolean condition) { + if (!condition) { + throw failure(TargetJdbcConnectionErrorCode.TARGET_MISMATCH); + } + } + + private static String expectedProduct(MetadataDatabaseKind kind) { + return kind == MetadataDatabaseKind.MYSQL ? "MySQL" : "PostgreSQL"; + } + + private static void closeAfterFailure( + Connection connection, TargetJdbcConnectionException original) { + boolean interrupted = Thread.interrupted(); + try { + connection.close(); + } catch (SQLException | RuntimeException cleanupFailure) { + throw failure(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED); + } catch (Error fatal) { + fatal.addSuppressed(original); + throw fatal; + } finally { + interrupted |= Thread.interrupted(); + restoreInterrupt(interrupted); + } + } + + private static void closeAfterFatal(Connection connection, Error original) { + boolean interrupted = Thread.interrupted(); + try { + connection.close(); + } catch (SQLException | RuntimeException cleanupFailure) { + // The original fatal failure retains priority and no JDBC diagnostic is attached. + } catch (Error cleanupFatal) { + // Do not attach a second possibly sensitive fatal diagnostic. + } finally { + interrupted |= Thread.interrupted(); + restoreInterrupt(interrupted); + } + } + + private static void restoreInterrupt(boolean interrupted) { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + private static TargetJdbcConnectionException failure(TargetJdbcConnectionErrorCode code) { + return new TargetJdbcConnectionException(code); + } + + @FunctionalInterface + private interface JdbcCall { + T execute() throws SQLException; + } + + @FunctionalInterface + private interface JdbcAction { + void execute() throws SQLException; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnector.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnector.java new file mode 100644 index 0000000000..e821ccafc0 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnector.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.sql.SQLException; + +/** Opens one provisional target connection without retaining its credentials. */ +@FunctionalInterface +interface TargetJdbcConnector { + + Connection connect( + TargetJdbcUrl target, + String username, + char[] password, + JdbcMetadataMigrationDeadline deadline) throws SQLException; +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcDataSourceProvider.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcDataSourceProvider.java new file mode 100644 index 0000000000..3553d5800c --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcDataSourceProvider.java @@ -0,0 +1,18 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.SQLException; +import javax.sql.DataSource; + +/** Builds a credential-free vendor DataSource for one connection attempt. */ +@FunctionalInterface +interface TargetJdbcDataSourceProvider { + + DataSource create(TargetJdbcDataSourceSettings settings) throws SQLException; +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcDataSourceSettings.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcDataSourceSettings.java new file mode 100644 index 0000000000..112627cb8f --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcDataSourceSettings.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; +import java.time.Duration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Credential-free vendor DataSource settings for one connection attempt. */ +record TargetJdbcDataSourceSettings( + MetadataDatabaseKind kind, + String jdbcUrl, + Duration remaining) { + + TargetJdbcDataSourceSettings { + Objects.requireNonNull(kind, "kind"); + Objects.requireNonNull(jdbcUrl, "jdbcUrl"); + Objects.requireNonNull(remaining, "remaining"); + if (remaining.isZero() || remaining.isNegative()) { + throw new IllegalArgumentException("Invalid target JDBC timeout settings"); + } + } + + @Override + public String toString() { + return "TargetJdbcDataSourceSettings[kind=" + kind + ']'; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcEndpoint.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcEndpoint.java new file mode 100644 index 0000000000..464a729991 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcEndpoint.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.net.URI; +import java.util.Locale; +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Ephemeral endpoint-only view of a driver-reported JDBC URL. */ +final class TargetJdbcEndpoint { + + private final MetadataDatabaseKind kind; + private final String host; + private final int port; + private final String database; + + private TargetJdbcEndpoint( + MetadataDatabaseKind kind, String host, int port, String database) { + this.kind = kind; + this.host = host; + this.port = port; + this.database = database; + } + + static TargetJdbcEndpoint parse(MetadataDatabaseKind kind, String actualUrl) { + try { + return parseStrict( + Objects.requireNonNull(kind, "kind"), + Objects.requireNonNull(actualUrl, "actualUrl")); + } catch (RuntimeException invalid) { + throw new IllegalArgumentException("Invalid target JDBC endpoint"); + } + } + + boolean matches(TargetJdbcUrl configured) { + return configured != null + && kind == configured.kind() + && host.equals(configured.host()) + && port == configured.port() + && database.equals(configured.database()); + } + + @Override + public String toString() { + return "TargetJdbcEndpoint[kind=" + kind + ']'; + } + + private static TargetJdbcEndpoint parseStrict( + MetadataDatabaseKind kind, String actualUrl) { + String prefix = switch (kind) { + case MYSQL -> "jdbc:mysql://"; + case POSTGRESQL -> "jdbc:postgresql://"; + case H2 -> throw new IllegalArgumentException(); + }; + if (!actualUrl.startsWith(prefix)) { + throw new IllegalArgumentException(); + } + URI uri = URI.create(actualUrl.substring("jdbc:".length())); + String scheme = kind == MetadataDatabaseKind.MYSQL ? "mysql" : "postgresql"; + if (!scheme.equals(uri.getScheme()) || uri.getRawUserInfo() != null || uri.getFragment() != null) { + throw new IllegalArgumentException(); + } + String rawHost = uri.getHost(); + int configuredPort = uri.getPort(); + if (rawHost == null || rawHost.isBlank() || configuredPort == 0 || configuredPort > 65535) { + throw new IllegalArgumentException(); + } + String expectedAuthority = rawHost + (configuredPort < 0 ? "" : ":" + configuredPort); + if (!expectedAuthority.equalsIgnoreCase(uri.getRawAuthority())) { + throw new IllegalArgumentException(); + } + String rawPath = uri.getRawPath(); + if (rawPath == null || rawPath.length() < 2 || rawPath.charAt(0) != '/' + || rawPath.indexOf('/', 1) >= 0) { + throw new IllegalArgumentException(); + } + return new TargetJdbcEndpoint( + kind, + rawHost.toLowerCase(Locale.ROOT), + configuredPort < 0 ? TargetJdbcUrl.defaultPort(kind) : configuredPort, + TargetJdbcUrl.decodeDatabase(rawPath.substring(1))); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcIdentity.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcIdentity.java new file mode 100644 index 0000000000..07f18c8cc3 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcIdentity.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; +import java.util.Objects; + +/** Builds the credential-free, length-framed identity of one verified target connection. */ +final class TargetJdbcIdentity { + + private TargetJdbcIdentity() { + } + + static String hash(TargetJdbcUrl target, String username, String catalog, String schema) { + Objects.requireNonNull(target, "target"); + requireText(username); + requireText(catalog); + MessageDigest digest = sha256(); + add(digest, target.kind().name()); + add(digest, target.canonicalUrl()); + add(digest, username); + add(digest, catalog); + add(digest, schema); + return HexFormat.of().formatHex(digest.digest()); + } + + private static void add(MessageDigest digest, String value) { + if (value == null) { + digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(-1).array()); + return; + } + byte[] encoded = value.getBytes(StandardCharsets.UTF_8); + digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(encoded.length).array()); + digest.update(encoded); + } + + private static MessageDigest sha256() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException unavailable) { + throw new IllegalStateException("SHA-256 unavailable"); + } + } + + private static void requireText(String value) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException("Invalid target JDBC identity"); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcResultWaiter.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcResultWaiter.java new file mode 100644 index 0000000000..ddcb2ef370 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcResultWaiter.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +/** Wait boundary used to arbitrate deadline expiry against a published result. */ +@FunctionalInterface +interface TargetJdbcResultWaiter { + + TargetJdbcResultWaiter TIMED = (ready, remainingNanos) -> + ready.await(remainingNanos, TimeUnit.NANOSECONDS); + + boolean await(CountDownLatch ready, long remainingNanos) throws InterruptedException; +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcUrl.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcUrl.java new file mode 100644 index 0000000000..f0d50f2987 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcUrl.java @@ -0,0 +1,263 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.ByteArrayOutputStream; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Set; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Strict, credential-free identity view of one supported single-host target JDBC URL. */ +final class TargetJdbcUrl { + + private static final Set USER_KEYS = Set.of("user", "username"); + + private final MetadataDatabaseKind kind; + private final String connectionUrl; + private final String canonicalUrl; + private final String host; + private final int port; + private final String database; + + private TargetJdbcUrl( + MetadataDatabaseKind kind, + String connectionUrl, + String canonicalUrl, + String host, + int port, + String database) { + this.kind = kind; + this.connectionUrl = connectionUrl; + this.canonicalUrl = canonicalUrl; + this.host = host; + this.port = port; + this.database = database; + } + + static TargetJdbcUrl parse(MetadataDatabaseKind kind, String jdbcUrl) { + try { + return parseStrict(Objects.requireNonNull(kind, "kind"), Objects.requireNonNull(jdbcUrl, "jdbcUrl")); + } catch (RuntimeException invalid) { + throw invalidUrl(); + } + } + + String canonicalUrl() { + return canonicalUrl; + } + + String connectionUrl() { + return connectionUrl; + } + + MetadataDatabaseKind kind() { + return kind; + } + + String host() { + return host; + } + + int port() { + return port; + } + + String database() { + return database; + } + + boolean sameTarget(TargetJdbcUrl other) { + return other != null + && kind == other.kind + && host.equals(other.host) + && port == other.port + && database.equals(other.database); + } + + @Override + public String toString() { + return "TargetJdbcUrl[kind=" + kind + "]"; + } + + private static TargetJdbcUrl parseStrict(MetadataDatabaseKind kind, String jdbcUrl) { + String prefix = switch (kind) { + case MYSQL -> "jdbc:mysql://"; + case POSTGRESQL -> "jdbc:postgresql://"; + case H2 -> throw invalidUrl(); + }; + if (!jdbcUrl.startsWith(prefix)) { + throw invalidUrl(); + } + + URI uri = URI.create(jdbcUrl.substring("jdbc:".length())); + String scheme = kind == MetadataDatabaseKind.MYSQL ? "mysql" : "postgresql"; + if (!scheme.equals(uri.getScheme()) || uri.getRawUserInfo() != null || uri.getFragment() != null) { + throw invalidUrl(); + } + + String rawHost = uri.getHost(); + int configuredPort = uri.getPort(); + if (rawHost == null || rawHost.isBlank() || configuredPort == 0 || configuredPort > 65535) { + throw invalidUrl(); + } + String expectedAuthority = rawHost + (configuredPort < 0 ? "" : ":" + configuredPort); + if (!expectedAuthority.equalsIgnoreCase(uri.getRawAuthority())) { + throw invalidUrl(); + } + + String rawPath = uri.getRawPath(); + if (rawPath == null || rawPath.length() < 2 || rawPath.charAt(0) != '/' + || rawPath.indexOf('/', 1) >= 0) { + throw invalidUrl(); + } + String database = decodeDatabase(rawPath.substring(1)); + + String host = rawHost.toLowerCase(Locale.ROOT); + int port = configuredPort < 0 ? defaultPort(kind) : configuredPort; + String query = canonicalQuery(uri.getRawQuery()); + String canonical = "jdbc:" + scheme + "://" + host + ':' + port + '/' + + encode(database) + (query.isEmpty() ? "" : "?" + query); + return new TargetJdbcUrl(kind, jdbcUrl, canonical, host, port, database); + } + + private static String canonicalQuery(String rawQuery) { + if (rawQuery == null) { + return ""; + } + if (rawQuery.isEmpty()) { + throw invalidUrl(); + } + Set keys = new HashSet<>(); + List parameters = new ArrayList<>(); + for (String part : rawQuery.split("&", -1)) { + int separator = part.indexOf('='); + if (separator <= 0) { + throw invalidUrl(); + } + String key = decode(part.substring(0, separator)).toLowerCase(Locale.ROOT); + String value = decode(part.substring(separator + 1)); + if (key.isBlank() || containsControl(key) || containsControl(value) + || credentialKey(key) || !keys.add(key)) { + throw invalidUrl(); + } + parameters.add(new QueryParameter(key, value)); + } + parameters.sort(Comparator.comparing(QueryParameter::key)); + return parameters.stream() + .map(parameter -> encode(parameter.key()) + '=' + encode(parameter.value())) + .reduce((left, right) -> left + '&' + right) + .orElseThrow(TargetJdbcUrl::invalidUrl); + } + + private static String decode(String raw) { + StringBuilder decoded = new StringBuilder(raw.length()); + for (int index = 0; index < raw.length();) { + if (raw.charAt(index) != '%') { + int codePoint = raw.codePointAt(index); + decoded.appendCodePoint(codePoint); + index += Character.charCount(codePoint); + continue; + } + ByteArrayOutputStream escaped = new ByteArrayOutputStream(); + while (index < raw.length() && raw.charAt(index) == '%') { + if (index + 2 >= raw.length()) { + throw invalidUrl(); + } + int high = Character.digit(raw.charAt(index + 1), 16); + int low = Character.digit(raw.charAt(index + 2), 16); + if (high < 0 || low < 0) { + throw invalidUrl(); + } + escaped.write(high << 4 | low); + index += 3; + } + decoded.append(decodeUtf8(escaped.toByteArray())); + } + return decoded.toString(); + } + + private static String decodeUtf8(byte[] encoded) { + try { + CharBuffer decoded = StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(encoded)); + return decoded.toString(); + } catch (CharacterCodingException invalid) { + throw invalidUrl(); + } + } + + private static String encode(String value) { + StringBuilder encoded = new StringBuilder(value.length()); + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + for (byte current : bytes) { + int unsigned = current & 0xff; + if (isUnreserved(unsigned)) { + encoded.append((char) unsigned); + } else { + encoded.append('%'); + encoded.append(Character.toUpperCase(Character.forDigit(unsigned >>> 4, 16))); + encoded.append(Character.toUpperCase(Character.forDigit(unsigned & 0x0f, 16))); + } + } + return encoded.toString(); + } + + private static boolean isUnreserved(int value) { + return value >= 'a' && value <= 'z' + || value >= 'A' && value <= 'Z' + || value >= '0' && value <= '9' + || value == '-' || value == '.' || value == '_' || value == '~'; + } + + private static boolean containsStructuralCharacter(String value) { + return containsControl(value) || value.indexOf('/') >= 0 || value.indexOf('?') >= 0 || value.indexOf('#') >= 0; + } + + private static boolean containsControl(String value) { + return value.codePoints().anyMatch(Character::isISOControl); + } + + private static boolean credentialKey(String key) { + return USER_KEYS.contains(key) + || key.contains("password") + || key.contains("secret") + || key.contains("token"); + } + + static String decodeDatabase(String rawDatabase) { + String database = decode(rawDatabase); + if (database.isBlank() || containsStructuralCharacter(database)) { + throw invalidUrl(); + } + return database; + } + + static int defaultPort(MetadataDatabaseKind kind) { + return kind == MetadataDatabaseKind.MYSQL ? 3306 : 5432; + } + + private static IllegalArgumentException invalidUrl() { + return new IllegalArgumentException("Invalid target JDBC URL"); + } + + private record QueryParameter(String key, String value) { + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcVendorConnector.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcVendorConnector.java new file mode 100644 index 0000000000..8713153905 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcVendorConnector.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import com.mysql.cj.jdbc.MysqlDataSource; +import java.sql.Connection; +import java.sql.SQLException; +import java.time.Duration; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import javax.sql.DataSource; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.postgresql.ds.PGSimpleDataSource; + +/** Opens target connections through vendor DataSources with a finite network budget. */ +final class TargetJdbcVendorConnector implements TargetJdbcConnector { + + private final TargetJdbcDataSourceProvider dataSources; + + TargetJdbcVendorConnector() { + this(TargetJdbcVendorConnector::createDataSource); + } + + TargetJdbcVendorConnector(TargetJdbcDataSourceProvider dataSources) { + this.dataSources = Objects.requireNonNull(dataSources, "dataSources"); + } + + @Override + public Connection connect( + TargetJdbcUrl target, + String username, + char[] password, + JdbcMetadataMigrationDeadline deadline) throws SQLException { + Objects.requireNonNull(target, "target"); + Objects.requireNonNull(username, "username"); + Objects.requireNonNull(password, "password"); + Duration remaining = remaining(deadline); + TargetJdbcDataSourceSettings settings = new TargetJdbcDataSourceSettings( + target.kind(), target.connectionUrl(), remaining); + DataSource dataSource = dataSources.create(settings); + remaining(deadline); + String ephemeralPassword = new String(password); + return dataSource.getConnection(username, ephemeralPassword); + } + + private static Duration remaining(JdbcMetadataMigrationDeadline deadline) { + try { + return deadline.remainingDuration(); + } catch (MetadataMigrationException deadlineFailure) { + if (deadlineFailure.code() == MetadataMigrationErrorCode.TIMEOUT) { + throw new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.TIMEOUT); + } + throw deadlineFailure; + } + } + + private static int positiveCeiling(long durationNanos, long unitNanos) { + long units = durationNanos / unitNanos; + if (durationNanos % unitNanos != 0) { + units++; + } + return (int) Math.min(Integer.MAX_VALUE, Math.max(1, units)); + } + + static DataSource createDataSource(TargetJdbcDataSourceSettings settings) throws SQLException { + if (settings.kind() == MetadataDatabaseKind.MYSQL) { + int timeoutMillis = positiveCeiling( + settings.remaining().toNanos(), TimeUnit.MILLISECONDS.toNanos(1)); + MysqlDataSource dataSource = new MysqlDataSource(); + dataSource.setURL(settings.jdbcUrl()); + dataSource.setConnectTimeout(timeoutMillis); + dataSource.setSocketTimeout(timeoutMillis); + return dataSource; + } + int timeoutSeconds = positiveCeiling( + settings.remaining().toNanos(), TimeUnit.SECONDS.toNanos(1)); + PGSimpleDataSource dataSource = new PGSimpleDataSource(); + dataSource.setURL(settings.jdbcUrl()); + dataSource.setConnectTimeout(timeoutSeconds); + dataSource.setSocketTimeout(timeoutSeconds); + dataSource.setCancelSignalTimeout(timeoutSeconds); + return dataSource; + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcCleanupLaneTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcCleanupLaneTest.java new file mode 100644 index 0000000000..553e84e07f --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcCleanupLaneTest.java @@ -0,0 +1,512 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.lang.reflect.Proxy; +import java.sql.Connection; +import java.sql.SQLException; +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +@Timeout(15) +class TargetJdbcCleanupLaneTest { + + @Test + void initiallyExpiredRetryNeverSubmitsCleanupWork() throws Exception { + AtomicInteger submissions = new AtomicInteger(); + AtomicInteger ticker = new AtomicInteger(); + ThreadPoolExecutor worker = countingInlineWorker(submissions); + Connection connection = mock(Connection.class); + TargetJdbcCleanupLane lane = new TargetJdbcCleanupLane(worker, Runnable::run); + lane.poison(connection); + JdbcMetadataMigrationDeadline deadline = JdbcMetadataMigrationDeadline.start( + Duration.ofNanos(1), ticker::get); + ticker.set(2); + try { + assertCleanupRequired(() -> lane.retry(deadline)); + + assertThat(submissions).hasValue(0); + verify(connection, times(0)).close(); + assertThat(lane.acquisitionFailure()).isEqualTo(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED); + } finally { + lane.close(); + } + } + + @Test + void retryDoesNotSubmitAgainWhenTheBudgetExpiresAfterTransientRejection() throws Exception { + AtomicInteger submissions = new AtomicInteger(); + AtomicInteger tickerReads = new AtomicInteger(); + ThreadPoolExecutor worker = rejectFirstInlineWorker(submissions); + Connection connection = mock(Connection.class); + TargetJdbcCleanupLane lane = new TargetJdbcCleanupLane(worker, Runnable::run); + lane.poison(connection); + JdbcMetadataMigrationDeadline deadline = JdbcMetadataMigrationDeadline.start( + Duration.ofNanos(1), + () -> tickerReads.getAndIncrement() < 2 ? 0 : 2); + try { + assertCleanupRequired(() -> lane.retry(deadline)); + + assertThat(submissions).hasValue(1); + verify(connection, times(0)).close(); + assertThat(lane.acquisitionFailure()).isEqualTo(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED); + } finally { + lane.close(); + } + } + + @Test + void retryWaitsUntilTheZeroQueueWorkerHasFinishedAfterExecute() throws Exception { + CountDownLatch afterExecuteEntered = new CountDownLatch(1); + CountDownLatch releaseAfterExecute = new CountDownLatch(1); + CountDownLatch retryEntered = new CountDownLatch(1); + AtomicBoolean firstTask = new AtomicBoolean(true); + ThreadPoolExecutor worker = new ThreadPoolExecutor( + 0, 1, 30, TimeUnit.SECONDS, new SynchronousQueue<>(), + Thread.ofPlatform().daemon(true).name("target-jdbc-cleanup-handoff", 0).factory(), + new ThreadPoolExecutor.AbortPolicy()) { + @Override + protected void afterExecute(Runnable task, Throwable failure) { + if (firstTask.getAndSet(false)) { + afterExecuteEntered.countDown(); + awaitUninterruptibly(releaseAfterExecute); + } + } + }; + worker.allowCoreThreadTimeOut(true); + Connection connection = mock(Connection.class); + doThrow(new SQLException("private first cleanup")) + .doNothing() + .when(connection).close(); + TargetJdbcCleanupLane lane = new TargetJdbcCleanupLane(worker, Runnable::run); + ExecutorService caller = Executors.newSingleThreadExecutor(); + Future retry = null; + try { + lane.cleanupLate(connection); + assertThat(afterExecuteEntered.await(5, TimeUnit.SECONDS)).isTrue(); + retry = caller.submit(() -> { + retryEntered.countDown(); + lane.retry(deadline()); + }); + assertThat(retryEntered.await(5, TimeUnit.SECONDS)).isTrue(); + + assertThat(retry).isNotDone(); + verify(connection).close(); + + releaseAfterExecute.countDown(); + retry.get(5, TimeUnit.SECONDS); + verify(connection, times(2)).close(); + } finally { + releaseAfterExecute.countDown(); + if (retry != null) { + retry.cancel(true); + } + caller.shutdownNow(); + lane.close(); + worker.shutdownNow(); + } + } + + @Test + void closeHandsOffRetainedCleanupAfterZeroQueueWorkerLeavesAfterExecute() throws Exception { + CountDownLatch afterExecuteEntered = new CountDownLatch(1); + CountDownLatch releaseAfterExecute = new CountDownLatch(1); + CountDownLatch cleanupCompleted = new CountDownLatch(1); + AtomicBoolean firstTask = new AtomicBoolean(true); + ThreadPoolExecutor worker = new ThreadPoolExecutor( + 0, 1, 30, TimeUnit.SECONDS, new SynchronousQueue<>(), + Thread.ofPlatform().daemon(true).name("target-jdbc-cleanup-close-handoff", 0).factory(), + new ThreadPoolExecutor.AbortPolicy()) { + @Override + protected void afterExecute(Runnable task, Throwable failure) { + if (firstTask.getAndSet(false)) { + afterExecuteEntered.countDown(); + awaitUninterruptibly(releaseAfterExecute); + } + } + }; + worker.allowCoreThreadTimeOut(true); + Connection connection = mock(Connection.class); + doThrow(new SQLException("private first cleanup")) + .doAnswer(ignored -> { + cleanupCompleted.countDown(); + return null; + }) + .when(connection).close(); + TargetJdbcCleanupLane lane = new TargetJdbcCleanupLane(worker, Runnable::run); + try { + lane.cleanupLate(connection); + assertThat(afterExecuteEntered.await(5, TimeUnit.SECONDS)).isTrue(); + + lane.close(); + releaseAfterExecute.countDown(); + + assertThat(cleanupCompleted.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(lane.acquisitionFailure()).isEqualTo(TargetJdbcConnectionErrorCode.FACTORY_CLOSED); + verify(connection, times(2)).close(); + } finally { + releaseAfterExecute.countDown(); + lane.close(); + worker.shutdownNow(); + } + } + + @Test + void abortInterruptIsClearedBeforeCloseAndCombinedWithEntryAndCloseInterrupts() throws Exception { + Thread.interrupted(); + Connection connection = mock(Connection.class); + AtomicBoolean closeSawInterrupt = new AtomicBoolean(); + doAnswer(ignored -> { + Thread.currentThread().interrupt(); + return null; + }).when(connection).abort(any()); + doAnswer(ignored -> { + closeSawInterrupt.set(Thread.currentThread().isInterrupted()); + Thread.currentThread().interrupt(); + return null; + }).when(connection).close(); + try (TargetJdbcCleanupLane lane = new TargetJdbcCleanupLane(inlineWorker(), Runnable::run)) { + Thread.currentThread().interrupt(); + + lane.cleanupLate(connection); + + assertThat(closeSawInterrupt).isFalse(); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } finally { + Thread.interrupted(); + } + } + + @Test + void abortFailureInterruptIsClearedBeforeCloseWhileCloseFatalRemainsPrimary() throws Exception { + Thread.interrupted(); + Connection connection = mock(Connection.class); + AtomicBoolean closeSawInterrupt = new AtomicBoolean(); + AssertionError fatal = new AssertionError("private close fatal"); + doAnswer(ignored -> { + Thread.currentThread().interrupt(); + throw new SQLException("private abort failure"); + }).when(connection).abort(any()); + doAnswer(ignored -> { + closeSawInterrupt.set(Thread.currentThread().isInterrupted()); + Thread.currentThread().interrupt(); + throw fatal; + }).when(connection).close(); + try (TargetJdbcCleanupLane lane = new TargetJdbcCleanupLane(inlineWorker(), Runnable::run)) { + lane.cleanupLate(connection); + + assertThatThrownBy(() -> lane.retry(deadline())).isSameAs(fatal); + assertThat(closeSawInterrupt).isFalse(); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } finally { + Thread.interrupted(); + } + } + + @Test + void closeDrainsManyDistinctHandlesWithoutRecursiveTaskExecution() { + AtomicInteger nextExpected = new AtomicInteger(); + AtomicBoolean ordered = new AtomicBoolean(true); + try (TargetJdbcCleanupLane lane = new TargetJdbcCleanupLane(inlineWorker(), Runnable::run)) { + for (int index = 0; index < 20_000; index++) { + Connection connection = orderedConnection(index, nextExpected, ordered); + lane.poison(connection); + lane.poison(connection); + } + + lane.close(); + + assertThat(nextExpected).hasValue(20_000); + assertThat(ordered).isTrue(); + } + } + + @Test + void completedFatalIsReplayedBeforeAnyLaterRetryMutation() throws Exception { + Connection connection = mock(Connection.class); + AssertionError fatal = new AssertionError("private completed fatal"); + doThrow(fatal).doNothing().when(connection).close(); + try (TargetJdbcCleanupLane lane = new TargetJdbcCleanupLane(inlineWorker(), Runnable::run)) { + lane.cleanupLate(connection); + + assertThatThrownBy(() -> lane.retry(deadline())).isSameAs(fatal); + verify(connection).close(); + + lane.retry(deadline()); + verify(connection, times(2)).close(); + } + } + + @Test + void runtimeSubmissionFailureFinalizesStateAndKeepsExactHandleRetryable() throws Exception { + Connection connection = mock(Connection.class); + try (TargetJdbcCleanupLane lane = new TargetJdbcCleanupLane( + oneShotFailureWorker(new IllegalStateException("private submit runtime")), Runnable::run)) { + lane.cleanupLate(connection); + + assertThat(lane.acquisitionFailure()).isEqualTo(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED); + lane.retry(deadline()); + verify(connection).close(); + } + } + + @Test + void fatalSubmissionFailureIsReplayedAndKeepsExactHandleRetryable() throws Exception { + Connection connection = mock(Connection.class); + AssertionError fatal = new AssertionError("private submit fatal"); + try (TargetJdbcCleanupLane lane = new TargetJdbcCleanupLane( + oneShotFailureWorker(fatal), Runnable::run)) { + lane.cleanupLate(connection); + + assertThatThrownBy(() -> lane.retry(deadline())).isSameAs(fatal); + lane.retry(deadline()); + verify(connection).close(); + } + } + + @Test + void closeStartsExactRetainedCleanupAndKeepsItRetryableUntilSuccess() throws Exception { + Connection connection = mock(Connection.class); + CountDownLatch closed = new CountDownLatch(1); + doThrow(new SQLException("private retained cleanup")) + .doAnswer(ignored -> { + closed.countDown(); + return null; + }) + .when(connection).close(); + TargetJdbcCleanupLane lane = lane(); + lane.poison(connection); + + lane.close(); + + assertCleanupRequired(() -> lane.retry(deadline())); + lane.retry(deadline()); + assertThat(closed.await(5, TimeUnit.SECONDS)).isTrue(); + verify(connection, times(2)).close(); + } + + @Test + void twoDistinctPoisonedHandlesAreBothOwnedAndClosed() throws Exception { + Connection first = mock(Connection.class); + Connection second = mock(Connection.class); + CountDownLatch closed = new CountDownLatch(2); + doAnswer(ignored -> { + closed.countDown(); + return null; + }).when(first).close(); + doAnswer(ignored -> { + closed.countDown(); + return null; + }).when(second).close(); + TargetJdbcCleanupLane lane = lane(); + + lane.poison(first); + lane.poison(second); + lane.close(); + + assertThat(closed.await(5, TimeUnit.SECONDS)).isTrue(); + verify(first).close(); + verify(second).close(); + } + + @Test + void repeatedStableFailuresRetainTheExactHandleUntilCleanupConverges() throws Exception { + Connection connection = mock(Connection.class); + doThrow(new SQLException("private SQL cleanup")) + .doThrow(new IllegalStateException("private runtime cleanup")) + .doNothing() + .when(connection).close(); + try (TargetJdbcCleanupLane lane = lane()) { + lane.cleanupLate(connection); + + assertCleanupRequired(() -> lane.retry(deadline())); + lane.retry(deadline()); + + verify(connection).abort(any()); + verify(connection, times(3)).close(); + assertThat(lane.acquisitionFailure()).isEqualTo(TargetJdbcConnectionErrorCode.FACTORY_CLOSED); + } + } + + @Test + void fatalCleanupRemainsPrimaryWhileTheExactHandleCanStillBeRetried() throws Exception { + Connection connection = mock(Connection.class); + AssertionError fatal = new AssertionError("private cleanup fatal"); + doThrow(fatal).doNothing().when(connection).close(); + try (TargetJdbcCleanupLane lane = lane()) { + lane.cleanupLate(connection); + + assertThatThrownBy(() -> lane.retry(deadline())).isSameAs(fatal); + lane.retry(deadline()); + + verify(connection).abort(any()); + verify(connection, times(2)).close(); + } + } + + @Test + void retryWithoutAnOwnedHandleReturnsStableFactoryClosed() { + try (TargetJdbcCleanupLane lane = lane()) { + assertThatThrownBy(() -> lane.retry(deadline())) + .isInstanceOfSatisfying(TargetJdbcConnectionException.class, failure -> { + assertThat(failure.code()).isEqualTo(TargetJdbcConnectionErrorCode.FACTORY_CLOSED); + assertThat(failure).hasNoCause(); + }); + } + } + + @Test + void shutdownRejectsNewCleanupWorkWithoutLosingCleanupRequiredState() throws SQLException { + Connection connection = mock(Connection.class); + TargetJdbcCleanupLane lane = lane(); + lane.close(); + + lane.cleanupLate(connection); + + assertThat(lane.acquisitionFailure()).isEqualTo(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED); + assertCleanupRequired(() -> lane.retry(deadline())); + verify(connection, times(0)).close(); + } + + private static void assertCleanupRequired(Runnable action) { + assertThatThrownBy(action::run) + .isInstanceOfSatisfying(TargetJdbcConnectionException.class, failure -> { + assertThat(failure.code()).isEqualTo(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED); + assertThat(failure).hasNoCause(); + }); + } + + private static TargetJdbcCleanupLane lane() { + ThreadPoolExecutor worker = new ThreadPoolExecutor( + 0, 1, 30, TimeUnit.SECONDS, new SynchronousQueue<>(), + Thread.ofPlatform().daemon(true).name("target-jdbc-cleanup-unit", 0).factory(), + new ThreadPoolExecutor.AbortPolicy()); + worker.allowCoreThreadTimeOut(true); + return new TargetJdbcCleanupLane(worker, Runnable::run); + } + + private static ThreadPoolExecutor inlineWorker() { + return new ThreadPoolExecutor(0, 1, 30, TimeUnit.SECONDS, new SynchronousQueue<>()) { + @Override + public void execute(Runnable task) { + task.run(); + } + }; + } + + private static ThreadPoolExecutor countingInlineWorker(AtomicInteger submissions) { + return new ThreadPoolExecutor(0, 1, 30, TimeUnit.SECONDS, new SynchronousQueue<>()) { + @Override + public void execute(Runnable task) { + submissions.incrementAndGet(); + task.run(); + } + }; + } + + private static ThreadPoolExecutor rejectFirstInlineWorker(AtomicInteger submissions) { + return new ThreadPoolExecutor(0, 1, 30, TimeUnit.SECONDS, new SynchronousQueue<>()) { + @Override + public void execute(Runnable task) { + if (submissions.incrementAndGet() == 1) { + throw new RejectedExecutionException("transient test rejection"); + } + task.run(); + } + }; + } + + private static ThreadPoolExecutor oneShotFailureWorker(Throwable failure) { + AtomicBoolean first = new AtomicBoolean(true); + return new ThreadPoolExecutor(0, 1, 30, TimeUnit.SECONDS, new SynchronousQueue<>()) { + @Override + public void execute(Runnable task) { + if (first.getAndSet(false)) { + if (failure instanceof Error fatal) { + throw fatal; + } + throw (RuntimeException) failure; + } + task.run(); + } + }; + } + + private static Connection orderedConnection( + int index, + AtomicInteger nextExpected, + AtomicBoolean ordered) { + return (Connection) Proxy.newProxyInstance( + TargetJdbcCleanupLaneTest.class.getClassLoader(), + new Class[] {Connection.class}, + (proxy, method, arguments) -> { + if (method.getName().equals("close")) { + if (nextExpected.getAndIncrement() != index) { + ordered.set(false); + } + return null; + } + if (method.getName().equals("isClosed")) { + return false; + } + if (method.getName().equals("toString")) { + return "counting-connection"; + } + Class returnType = method.getReturnType(); + if (!returnType.isPrimitive()) { + return null; + } + if (returnType == boolean.class) { + return false; + } + if (returnType == char.class) { + return '\0'; + } + return 0; + }); + } + + private static void awaitUninterruptibly(CountDownLatch latch) { + boolean interrupted = false; + while (true) { + try { + latch.await(); + break; + } catch (InterruptedException ignored) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + private static JdbcMetadataMigrationDeadline deadline() { + return JdbcMetadataMigrationDeadline.start(Duration.ofSeconds(5), System::nanoTime); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionFactoryLifecycleTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionFactoryLifecycleTest.java new file mode 100644 index 0000000000..b325c324cc --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionFactoryLifecycleTest.java @@ -0,0 +1,349 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +@Timeout(15) +class TargetJdbcConnectionFactoryLifecycleTest { + + private static final Duration TIMEOUT = Duration.ofSeconds(5); + + @Test + void defaultFactoryRejectsConcurrentAcquireWithoutQueuing() throws Exception { + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + TargetJdbcConnector connector = (target, username, password, deadline) -> { + entered.countDown(); + awaitIgnoringInterrupt(release); + return mysqlConnection(); + }; + try (TargetJdbcConnectionFactory factory = new TargetJdbcConnectionFactory(connector, Runnable::run); + ExecutorService caller = Executors.newSingleThreadExecutor()) { + Future first = caller.submit(() -> { + try (SecretValue password = SecretValue.of("secret")) { + return factory.acquire(settings(), password, deadline()); + } + }); + try { + assertThat(entered.await(5, TimeUnit.SECONDS)).isTrue(); + assertFailure(factory, TargetJdbcConnectionErrorCode.OPERATION_CONFLICT); + } finally { + release.countDown(); + } + first.get(5, TimeUnit.SECONDS).close(); + } finally { + release.countDown(); + } + } + + @Test + void rejectedSubmissionClearsActiveSlotAndReturnsStableFailure() { + ThreadPoolExecutor rejectingWorker = worker(); + rejectingWorker.shutdown(); + try (TargetJdbcConnectionFactory factory = new TargetJdbcConnectionFactory( + rejectingWorker, worker(), Runnable::run, + (target, username, password, deadline) -> mysqlConnection(), + new TargetJdbcConnectionVerifier(Runnable::run))) { + assertFailure(factory, TargetJdbcConnectionErrorCode.FACTORY_CLOSED); + assertFailure(factory, TargetJdbcConnectionErrorCode.FACTORY_CLOSED); + } + } + + @Test + void unexpectedSubmissionFatalClearsTheActiveSlotAndRemainsPrimary() { + AssertionError fatal = new AssertionError("private submit fatal"); + AtomicBoolean first = new AtomicBoolean(true); + ThreadPoolExecutor rejectingWorker = new ThreadPoolExecutor( + 0, 1, 30, TimeUnit.SECONDS, new SynchronousQueue<>()) { + @Override + public void execute(Runnable task) { + if (first.getAndSet(false)) { + throw fatal; + } + task.run(); + } + }; + try (TargetJdbcConnectionFactory factory = new TargetJdbcConnectionFactory( + rejectingWorker, worker(), Runnable::run, + (target, username, password, deadline) -> mysqlConnection(), + new TargetJdbcConnectionVerifier(Runnable::run))) { + assertThatThrownBy(() -> acquire(factory)).isSameAs(fatal); + assertThatThrownBy(() -> acquire(factory)).isSameAs(fatal); + } + } + + @Test + void unexpectedSubmissionRuntimeIsRedactedAndPermanentlyClosesFactory() { + AtomicBoolean first = new AtomicBoolean(true); + ThreadPoolExecutor rejectingWorker = new ThreadPoolExecutor( + 0, 1, 30, TimeUnit.SECONDS, new SynchronousQueue<>()) { + @Override + public void execute(Runnable task) { + if (first.getAndSet(false)) { + throw new IllegalStateException("private submit runtime"); + } + task.run(); + } + }; + try (TargetJdbcConnectionFactory factory = new TargetJdbcConnectionFactory( + rejectingWorker, worker(), Runnable::run, + (target, username, password, deadline) -> mysqlConnection(), + new TargetJdbcConnectionVerifier(Runnable::run))) { + assertFailure(factory, TargetJdbcConnectionErrorCode.FACTORY_CLOSED); + assertFailure(factory, TargetJdbcConnectionErrorCode.FACTORY_CLOSED); + } + } + + @Test + void finishedFatalIsPublishedAsPrimaryAndPoisonsTheExactConnection() throws Exception { + Connection connection = mysqlConnection(); + AssertionError fatal = new AssertionError("private owner fatal"); + FailingAttemptOwner owner = new FailingAttemptOwner(fatal); + char[] password = "secret".toCharArray(); + TargetJdbcConnectionAttempt attempt = attempt(owner, connection, password); + + attempt.run(); + + assertThatThrownBy(attempt::await).isSameAs(fatal); + assertThat(owner.poisoned()).isSameAs(connection); + assertThat(password).containsOnly('\0'); + } + + @Test + void finishedRuntimeIsPublishedAsCauseFreeUnavailableAndPoisonsTheExactConnection() throws Exception { + Connection connection = mysqlConnection(); + FailingAttemptOwner owner = new FailingAttemptOwner( + new IllegalStateException("private owner runtime")); + TargetJdbcConnectionAttempt attempt = attempt(owner, connection, "secret".toCharArray()); + + attempt.run(); + + assertThatThrownBy(attempt::await) + .isInstanceOfSatisfying(TargetJdbcConnectionException.class, failure -> { + assertThat(failure.code()).isEqualTo(TargetJdbcConnectionErrorCode.UNAVAILABLE); + assertThat(failure).hasNoCause(); + }); + assertThat(owner.poisoned()).isSameAs(connection); + } + + @Test + void connectorSqlAndRuntimeFailuresAreCauseFreeAndDoNotPoisonTheFactory() { + AtomicBoolean sql = new AtomicBoolean(true); + TargetJdbcConnector connector = (target, username, password, deadline) -> { + if (sql.getAndSet(false)) { + throw new SQLException("private SQL diagnostic"); + } + throw new IllegalStateException("private runtime diagnostic"); + }; + try (TargetJdbcConnectionFactory factory = new TargetJdbcConnectionFactory(connector, Runnable::run)) { + assertFailure(factory, TargetJdbcConnectionErrorCode.UNAVAILABLE); + assertFailure(factory, TargetJdbcConnectionErrorCode.UNAVAILABLE); + } + } + + @Test + void interruptedCallerReturnsStableTimeoutWithItsInterruptRestored() throws Exception { + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + TargetJdbcConnector connector = (target, username, password, deadline) -> { + entered.countDown(); + awaitIgnoringInterrupt(release); + return mock(Connection.class); + }; + try (TargetJdbcConnectionFactory factory = new TargetJdbcConnectionFactory(connector, Runnable::run); + ExecutorService caller = Executors.newSingleThreadExecutor()) { + AtomicReference callerThread = new AtomicReference<>(); + Future result = caller.submit(() -> { + callerThread.set(Thread.currentThread()); + TargetJdbcConnectionErrorCode code = failureCode(() -> acquire(factory)); + return new InterruptedResult(code, Thread.currentThread().isInterrupted()); + }); + try { + assertThat(entered.await(5, TimeUnit.SECONDS)).isTrue(); + callerThread.get().interrupt(); + assertThat(result.get(5, TimeUnit.SECONDS)).isEqualTo( + new InterruptedResult(TargetJdbcConnectionErrorCode.TIMEOUT, true)); + } finally { + release.countDown(); + } + } finally { + release.countDown(); + } + } + + @Test + void closeAfterResultPublicationDoesNotTakeOwnershipOfTheReturnedLease() throws Exception { + Connection connection = mysqlConnection(); + AtomicReference factoryRef = new AtomicReference<>(); + TargetJdbcResultWaiter closeAfterPublication = (ready, remaining) -> { + assertThat(ready.await(remaining, TimeUnit.NANOSECONDS)).isTrue(); + factoryRef.get().close(); + return true; + }; + TargetJdbcConnectionFactory factory = new TargetJdbcConnectionFactory( + worker(), worker(), Runnable::run, + (target, username, password, deadline) -> connection, + new TargetJdbcConnectionVerifier(Runnable::run), closeAfterPublication); + factoryRef.set(factory); + try (factory; SecretValue password = SecretValue.of("secret")) { + TargetJdbcConnectionLease lease = factory.acquire(settings(), password, deadline()); + verify(connection, times(0)).close(); + lease.close(); + verify(connection).close(); + } + } + + private static TargetJdbcConnectionLease acquire(TargetJdbcConnectionFactory factory) { + try (SecretValue password = SecretValue.of("secret")) { + return factory.acquire(settings(), password, deadline()); + } + } + + private static TargetJdbcConnectionAttempt attempt( + TargetJdbcConnectionAttemptOwner owner, Connection connection, char[] password) { + TargetJdbcResultWaiter published = (ready, remaining) -> { + assertThat(ready.getCount()).isZero(); + return true; + }; + return new TargetJdbcConnectionAttempt( + owner, (target, username, ignored, deadline) -> connection, + new TargetJdbcConnectionVerifier(Runnable::run), + TargetJdbcUrl.parse(MetadataDatabaseKind.MYSQL, + "jdbc:mysql://db.example/hertzbeat"), + "operator", password, deadline(), published); + } + + private static void assertFailure( + TargetJdbcConnectionFactory factory, TargetJdbcConnectionErrorCode expected) { + assertThatThrownBy(() -> acquire(factory)) + .isInstanceOfSatisfying(TargetJdbcConnectionException.class, failure -> { + assertThat(failure.code()).isEqualTo(expected); + assertThat(failure).hasNoCause(); + }); + } + + private static TargetJdbcConnectionErrorCode failureCode(Runnable action) { + try { + action.run(); + throw new AssertionError("Expected target JDBC connection failure"); + } catch (TargetJdbcConnectionException failure) { + return failure.code(); + } + } + + private static MetadataDatabaseSettings settings() { + return new MetadataDatabaseSettings( + MetadataDatabaseKind.MYSQL, + "jdbc:mysql://db.example/hertzbeat?sslMode=REQUIRED", "operator"); + } + + private static JdbcMetadataMigrationDeadline deadline() { + return JdbcMetadataMigrationDeadline.start(TIMEOUT, System::nanoTime); + } + + private static ThreadPoolExecutor worker() { + ThreadPoolExecutor worker = new ThreadPoolExecutor( + 0, 1, 30, TimeUnit.SECONDS, new SynchronousQueue<>(), + Thread.ofPlatform().daemon(true).name("target-jdbc-lifecycle-test", 0).factory(), + new ThreadPoolExecutor.AbortPolicy()); + worker.allowCoreThreadTimeOut(true); + return worker; + } + + private static Connection mysqlConnection() throws SQLException { + Connection connection = mock(Connection.class); + DatabaseMetaData metadata = mock(DatabaseMetaData.class); + when(connection.getAutoCommit()).thenReturn(true); + when(connection.isReadOnly()).thenReturn(false); + when(connection.getMetaData()).thenReturn(metadata); + when(metadata.getDatabaseProductName()).thenReturn("MySQL"); + when(metadata.getURL()).thenReturn("jdbc:mysql://db.example/hertzbeat"); + when(connection.getCatalog()).thenReturn("hertzbeat"); + return connection; + } + + private static void awaitIgnoringInterrupt(CountDownLatch latch) { + boolean interrupted = false; + while (true) { + try { + latch.await(); + break; + } catch (InterruptedException ignored) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + private record InterruptedResult(TargetJdbcConnectionErrorCode code, boolean interrupted) { + } + + private static final class FailingAttemptOwner implements TargetJdbcConnectionAttemptOwner { + + private final Throwable failure; + private Connection poisoned; + + private FailingAttemptOwner(Throwable failure) { + this.failure = failure; + } + + @Override + public void poison(Connection connection) { + poisoned = connection; + } + + @Override + public void cleanupLate(Connection connection) { + throw new AssertionError("Unexpected late cleanup"); + } + + @Override + public void lateFatal(Error fatal, Connection connection) { + throw new AssertionError("Unexpected late fatal"); + } + + @Override + public void finished(TargetJdbcConnectionAttempt attempt) { + if (failure instanceof Error fatal) { + throw fatal; + } + throw (RuntimeException) failure; + } + + private Connection poisoned() { + return poisoned; + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionFactoryTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionFactoryTest.java new file mode 100644 index 0000000000..8c94fc3740 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionFactoryTest.java @@ -0,0 +1,535 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.mockito.Mockito; + +@Timeout(15) +class TargetJdbcConnectionFactoryTest { + + private static final Duration TIMEOUT = Duration.ofSeconds(5); + + @Test + void successfulAcquirePublishesExactLeaseAndClearsTheWorkerPasswordCopy() throws Exception { + Connection connection = mysqlConnection(); + AtomicReference workerPassword = new AtomicReference<>(); + TargetJdbcConnector connector = (target, username, password, deadline) -> { + assertThat(target.connectionUrl()).isEqualTo( + "jdbc:mysql://DB.Example/hertzbeat?sslMode=REQUIRED"); + assertThat(username).isEqualTo("operator"); + workerPassword.set(password); + return connection; + }; + try (TargetJdbcConnectionFactory factory = factory(connector); + SecretValue password = SecretValue.of("secret")) { + TargetJdbcConnectionLease lease = factory.acquire(settings(), password, deadline(TIMEOUT)); + + lease.withConnection(actual -> assertThat(actual).isSameAs(connection)); + assertThat(lease.targetIdentityHash()).matches("[0-9a-f]{64}"); + assertThat(workerPassword.get()).containsOnly('\0'); + TargetJdbcConnectionLease second = factory.acquire(settings(), password, deadline(TIMEOUT)); + second.close(); + lease.close(); + } + } + + @Test + void connectorAndVerifierShareOneDeadlineAndExpiredConnectorBudgetGatesAllVerifierCalls() + throws Exception { + Connection connection = mysqlConnection(); + CountDownLatch cleaned = new CountDownLatch(1); + Mockito.doAnswer(ignored -> { + cleaned.countDown(); + return null; + }).when(connection).close(); + AtomicLong ticker = new AtomicLong(); + JdbcMetadataMigrationDeadline root = JdbcMetadataMigrationDeadline.start( + Duration.ofNanos(20), ticker::get); + TargetJdbcConnector connector = (target, username, password, deadline) -> { + assertThat(deadline).isSameAs(root); + ticker.set(21L); + return connection; + }; + try (TargetJdbcConnectionFactory factory = factory(connector); + SecretValue password = SecretValue.of("secret")) { + assertThatThrownBy(() -> factory.acquire(settings(), password, root)) + .isInstanceOfSatisfying(TargetJdbcConnectionException.class, + failure -> assertThat(failure.code()).isEqualTo(TargetJdbcConnectionErrorCode.TIMEOUT)); + assertThat(cleaned.await(5, TimeUnit.SECONDS)).isTrue(); + } + + verify(connection, times(0)).getAutoCommit(); + verify(connection).close(); + } + + @Test + void timedOutConnectCannotPublishAndLateConnectionIsAbortedAndClosedBeforeReturn() throws Exception { + Connection connection = mock(Connection.class); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch cleaned = new CountDownLatch(1); + AtomicLong ticker = new AtomicLong(); + Mockito.doAnswer(ignored -> { + cleaned.countDown(); + return null; + }).when(connection).close(); + TargetJdbcConnector connector = (target, username, password, deadline) -> { + ticker.set(21L); + entered.countDown(); + awaitIgnoringInterrupt(release); + return connection; + }; + try (TargetJdbcConnectionFactory factory = factory(connector); + ExecutorService caller = Executors.newSingleThreadExecutor(); + SecretValue password = SecretValue.of("secret")) { + Future result = caller.submit(() -> failureCode( + () -> factory.acquire(settings(), password, + JdbcMetadataMigrationDeadline.start(Duration.ofNanos(20), ticker::get)))); + assertThat(entered.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(result.get(5, TimeUnit.SECONDS)).isEqualTo(TargetJdbcConnectionErrorCode.TIMEOUT); + verify(connection, times(0)).close(); + release.countDown(); + assertThat(cleaned.await(5, TimeUnit.SECONDS)).isTrue(); + } + + verify(connection).abort(any()); + verify(connection).close(); + } + + @Test + void resultPublishedBeforeAbandonmentWinsIsReplayedInsteadOfLeakedAsTimeout() throws Exception { + Connection connection = mysqlConnection(); + TargetJdbcResultWaiter lateTimeout = (ready, ignored) -> { + assertThat(ready.await(5, TimeUnit.SECONDS)).isTrue(); + return false; + }; + try (TargetJdbcConnectionFactory factory = new TargetJdbcConnectionFactory( + worker(), cleanupWorker(), Runnable::run, + (target, username, password, deadline) -> connection, + new TargetJdbcConnectionVerifier(Runnable::run), lateTimeout); + SecretValue password = SecretValue.of("secret")) { + TargetJdbcConnectionLease lease = factory.acquire(settings(), password, deadline(TIMEOUT)); + + lease.withConnection(actual -> assertThat(actual).isSameAs(connection)); + lease.close(); + } + } + + @Test + void lateCleanupFailureIsVisibleAndRetainedForExactRetry() throws Exception { + Connection connection = mock(Connection.class); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch cleanupAttempted = new CountDownLatch(1); + AtomicLong ticker = new AtomicLong(); + AtomicInteger closes = new AtomicInteger(); + Mockito.doAnswer(ignored -> { + if (closes.incrementAndGet() == 1) { + cleanupAttempted.countDown(); + throw new SQLException("private late cleanup"); + } + return null; + }).when(connection).close(); + TargetJdbcConnector connector = (target, username, password, deadline) -> { + ticker.set(21L); + entered.countDown(); + awaitIgnoringInterrupt(release); + return connection; + }; + try (TargetJdbcConnectionFactory factory = factory(connector); + SecretValue password = SecretValue.of("secret")) { + assertThatThrownBy(() -> factory.acquire(settings(), password, + JdbcMetadataMigrationDeadline.start(Duration.ofNanos(20), ticker::get))) + .isInstanceOfSatisfying(TargetJdbcConnectionException.class, + failure -> assertThat(failure.code()).isEqualTo(TargetJdbcConnectionErrorCode.TIMEOUT)); + assertThat(entered.await(5, TimeUnit.SECONDS)).isTrue(); + release.countDown(); + assertThat(cleanupAttempted.await(5, TimeUnit.SECONDS)).isTrue(); + + awaitFailure(factory, TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED); + factory.retryCleanup(deadline(TIMEOUT)); + verify(connection, times(2)).close(); + assertFailure(factory, TargetJdbcConnectionErrorCode.FACTORY_CLOSED); + } + } + + @Test + void verifierCleanupFailurePoisonsFactoryAndRetainsExactCleanupForRetry() throws Exception { + Connection connection = connectionWith( + "MySQL", "jdbc:mysql://other.example/hertzbeat", "hertzbeat", null); + doThrow(new SQLException("private cleanup path")).doNothing().when(connection).close(); + AtomicInteger connects = new AtomicInteger(); + TargetJdbcConnector connector = (target, username, password, deadline) -> { + connects.incrementAndGet(); + return connection; + }; + try (TargetJdbcConnectionFactory factory = factory(connector)) { + assertFailure(factory, TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED); + assertFailure(factory, TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED); + assertThat(connects).hasValue(1); + + factory.retryCleanup(deadline(TIMEOUT)); + verify(connection, times(2)).close(); + assertFailure(factory, TargetJdbcConnectionErrorCode.FACTORY_CLOSED); + } + } + + @Test + void cleanupErrorPoisonsFactoryWithoutLosingTheExactConnection() throws Exception { + Connection connection = connectionWith( + "MySQL", "jdbc:mysql://other.example/hertzbeat", "hertzbeat", null); + AssertionError fatal = new AssertionError("private cleanup fatal"); + doThrow(fatal).doNothing().when(connection).close(); + try (TargetJdbcConnectionFactory factory = factory( + (target, username, password, deadline) -> connection); + SecretValue password = SecretValue.of("secret")) { + assertThatThrownBy(() -> factory.acquire(settings(), password, deadline(TIMEOUT))).isSameAs(fatal); + assertFailure(factory, TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED); + factory.retryCleanup(deadline(TIMEOUT)); + verify(connection, times(2)).close(); + } + } + + @Test + void blockedCleanupRetryReturnsAtDeadlineAndRetainsExactHandleUntilItConverges() throws Exception { + Connection connection = connectionWith( + "MySQL", "jdbc:mysql://other.example/hertzbeat", "hertzbeat", null); + CountDownLatch cleanupEntered = new CountDownLatch(1); + CountDownLatch releaseCleanup = new CountDownLatch(1); + CountDownLatch cleanupFinished = new CountDownLatch(1); + AtomicInteger closes = new AtomicInteger(); + AtomicLong ticker = new AtomicLong(); + Mockito.doAnswer(ignored -> { + int current = closes.incrementAndGet(); + if (current == 1) { + throw new SQLException("private verification cleanup"); + } + cleanupEntered.countDown(); + ticker.set(21L); + awaitIgnoringInterrupt(releaseCleanup); + cleanupFinished.countDown(); + return null; + }).when(connection).close(); + try (TargetJdbcConnectionFactory factory = factory( + (target, username, password, deadline) -> connection)) { + assertFailure(factory, TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED); + + assertThatThrownBy(() -> factory.retryCleanup( + JdbcMetadataMigrationDeadline.start(Duration.ofNanos(20), ticker::get))) + .isInstanceOfSatisfying(TargetJdbcConnectionException.class, + failure -> assertThat(failure.code()) + .isEqualTo(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED)); + assertThat(cleanupEntered.await(5, TimeUnit.SECONDS)).isTrue(); + assertFailure(factory, TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED); + releaseCleanup.countDown(); + assertThat(cleanupFinished.await(5, TimeUnit.SECONDS)).isTrue(); + awaitFailure(factory, TargetJdbcConnectionErrorCode.FACTORY_CLOSED); + verify(connection, times(2)).close(); + } finally { + releaseCleanup.countDown(); + } + } + + @Test + void fatalBeforeConnectionPoisonsFactory() { + AssertionError fatal = new AssertionError("private provider fatal"); + try (TargetJdbcConnectionFactory factory = factory( + (target, username, password, deadline) -> { throw fatal; }); + SecretValue password = SecretValue.of("secret")) { + assertThatThrownBy(() -> factory.acquire(settings(), password, deadline(TIMEOUT))).isSameAs(fatal); + assertFailure(factory, TargetJdbcConnectionErrorCode.FACTORY_CLOSED); + } + } + + @Test + void fatalAfterTimeoutIsRetainedInTheOwnedWorkerFailureChannel() throws Exception { + AssertionError fatal = new AssertionError("private late provider fatal"); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch exited = new CountDownLatch(1); + AtomicLong ticker = new AtomicLong(); + TargetJdbcConnector connector = (target, username, password, deadline) -> { + ticker.set(21L); + entered.countDown(); + try { + awaitIgnoringInterrupt(release); + throw fatal; + } finally { + exited.countDown(); + } + }; + try (TargetJdbcConnectionFactory factory = factory(connector); + SecretValue password = SecretValue.of("secret")) { + assertThatThrownBy(() -> factory.acquire(settings(), password, + JdbcMetadataMigrationDeadline.start(Duration.ofNanos(20), ticker::get))) + .isInstanceOfSatisfying(TargetJdbcConnectionException.class, + failure -> assertThat(failure.code()).isEqualTo(TargetJdbcConnectionErrorCode.TIMEOUT)); + assertThat(entered.await(5, TimeUnit.SECONDS)).isTrue(); + release.countDown(); + assertThat(exited.await(5, TimeUnit.SECONDS)).isTrue(); + assertThatThrownBy(() -> factory.acquire(settings(), password, deadline(TIMEOUT))).isSameAs(fatal); + } finally { + release.countDown(); + } + } + + @Test + void factoryRejectsConcurrentAcquireAndPermanentlyRejectsAfterClose() throws Exception { + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + TargetJdbcConnector connector = (target, username, password, deadline) -> { + entered.countDown(); + awaitIgnoringInterrupt(release); + return mysqlConnection(); + }; + TargetJdbcConnectionFactory factory = factory(connector); + try (ExecutorService caller = Executors.newSingleThreadExecutor()) { + Future active = caller.submit(() -> { + try (SecretValue password = SecretValue.of("secret")) { + return factory.acquire(settings(), password, deadline(TIMEOUT)); + } + }); + try { + assertThat(entered.await(5, TimeUnit.SECONDS)).isTrue(); + assertFailure(factory, TargetJdbcConnectionErrorCode.OPERATION_CONFLICT); + } finally { + release.countDown(); + } + TargetJdbcConnectionLease lease = (TargetJdbcConnectionLease) active.get(5, TimeUnit.SECONDS); + lease.close(); + } + factory.close(); + assertFailure(factory, TargetJdbcConnectionErrorCode.FACTORY_CLOSED); + } + + @Test + void closeWakesActiveCallerWithoutWaitingForStuckConnectAndCleansLateConnection() throws Exception { + Connection connection = mock(Connection.class); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch closed = new CountDownLatch(1); + Mockito.doAnswer(ignored -> { + closed.countDown(); + return null; + }).when(connection).close(); + TargetJdbcConnector connector = (target, username, password, deadline) -> { + entered.countDown(); + awaitIgnoringInterrupt(release); + return connection; + }; + TargetJdbcConnectionFactory factory = factory(connector); + try (ExecutorService caller = Executors.newSingleThreadExecutor(); + SecretValue password = SecretValue.of("secret")) { + Future result = caller.submit(() -> failureCode( + () -> factory.acquire(settings(), password, deadline(TIMEOUT)))); + assertThat(entered.await(5, TimeUnit.SECONDS)).isTrue(); + + factory.close(); + assertThat(result.get(5, TimeUnit.SECONDS)).isEqualTo(TargetJdbcConnectionErrorCode.FACTORY_CLOSED); + verify(connection, times(0)).close(); + release.countDown(); + assertThat(closed.await(5, TimeUnit.SECONDS)).isTrue(); + } finally { + release.countDown(); + factory.close(); + } + verify(connection).abort(any()); + verify(connection).close(); + } + + @Test + void lateCleanupClearsWorkerInterruptDuringDriverCallsAndRestoresItAfterward() throws Exception { + Connection connection = mock(Connection.class); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch cleanupExited = new CountDownLatch(1); + AtomicBoolean restored = new AtomicBoolean(); + AtomicLong ticker = new AtomicLong(); + ThreadPoolExecutor interruptedCleanupWorker = new ThreadPoolExecutor( + 0, 1, 30, TimeUnit.SECONDS, new SynchronousQueue<>(), + Thread.ofPlatform().daemon(true).name("target-jdbc-interrupt-cleanup", 0).factory(), + new ThreadPoolExecutor.AbortPolicy()) { + @Override + protected void beforeExecute(Thread thread, Runnable task) { + thread.interrupt(); + } + + @Override + protected void afterExecute(Runnable task, Throwable failure) { + restored.set(Thread.currentThread().isInterrupted()); + cleanupExited.countDown(); + } + }; + interruptedCleanupWorker.allowCoreThreadTimeOut(true); + Mockito.doAnswer(ignored -> { + assertThat(Thread.currentThread().isInterrupted()).isFalse(); + return null; + }).when(connection).abort(any()); + Mockito.doAnswer(ignored -> { + assertThat(Thread.currentThread().isInterrupted()).isFalse(); + Thread.currentThread().interrupt(); + return null; + }).when(connection).close(); + TargetJdbcConnector connector = (target, username, password, deadline) -> { + ticker.set(21L); + entered.countDown(); + awaitIgnoringInterrupt(release); + return connection; + }; + try (TargetJdbcConnectionFactory factory = new TargetJdbcConnectionFactory( + worker(), interruptedCleanupWorker, Runnable::run, connector, + new TargetJdbcConnectionVerifier(Runnable::run)); + SecretValue password = SecretValue.of("secret")) { + assertThatThrownBy(() -> factory.acquire(settings(), password, + JdbcMetadataMigrationDeadline.start(Duration.ofNanos(20), ticker::get))) + .isInstanceOf(TargetJdbcConnectionException.class); + assertThat(entered.await(5, TimeUnit.SECONDS)).isTrue(); + release.countDown(); + assertThat(cleanupExited.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(restored).isTrue(); + } finally { + release.countDown(); + interruptedCleanupWorker.shutdownNow(); + } + } + + private static TargetJdbcConnectionFactory factory(TargetJdbcConnector connector) { + return new TargetJdbcConnectionFactory( + worker(), cleanupWorker(), Runnable::run, connector, + new TargetJdbcConnectionVerifier(Runnable::run)); + } + + private static ThreadPoolExecutor worker() { + ThreadPoolExecutor worker = new ThreadPoolExecutor( + 0, 1, 30, TimeUnit.SECONDS, new SynchronousQueue<>(), + Thread.ofPlatform().daemon(true).name("target-jdbc-test", 0).factory(), + new ThreadPoolExecutor.AbortPolicy()); + worker.allowCoreThreadTimeOut(true); + return worker; + } + + private static ThreadPoolExecutor cleanupWorker() { + ThreadPoolExecutor worker = new ThreadPoolExecutor( + 0, 1, 30, TimeUnit.SECONDS, new SynchronousQueue<>(), + Thread.ofPlatform().daemon(true).name("target-jdbc-cleanup-test", 0).factory(), + new ThreadPoolExecutor.AbortPolicy()); + worker.allowCoreThreadTimeOut(true); + return worker; + } + + private static MetadataDatabaseSettings settings() { + return new MetadataDatabaseSettings( + MetadataDatabaseKind.MYSQL, + "jdbc:mysql://DB.Example/hertzbeat?sslMode=REQUIRED", "operator"); + } + + private static JdbcMetadataMigrationDeadline deadline(Duration timeout) { + return JdbcMetadataMigrationDeadline.start(timeout, System::nanoTime); + } + + private static void assertFailure( + TargetJdbcConnectionFactory factory, TargetJdbcConnectionErrorCode expected) { + assertThatThrownBy(() -> { + try (SecretValue password = SecretValue.of("secret")) { + factory.acquire(settings(), password, deadline(TIMEOUT)); + } + }) + .isInstanceOfSatisfying(TargetJdbcConnectionException.class, failure -> { + assertThat(failure.code()).isEqualTo(expected); + assertThat(failure).hasNoCause(); + }); + } + + private static void awaitFailure( + TargetJdbcConnectionFactory factory, TargetJdbcConnectionErrorCode expected) { + long limit = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (true) { + try { + assertFailure(factory, expected); + return; + } catch (AssertionError notReady) { + if (System.nanoTime() >= limit) { + throw notReady; + } + Thread.onSpinWait(); + } + } + } + + private static TargetJdbcConnectionErrorCode failureCode(Runnable action) { + try { + action.run(); + throw new AssertionError("Expected target JDBC connection failure"); + } catch (TargetJdbcConnectionException failure) { + return failure.code(); + } + } + + private static Connection mysqlConnection() throws SQLException { + return connectionWith( + "MySQL", "jdbc:mysql://db.example/hertzbeat?sslmode=required", "hertzbeat", null); + } + + private static Connection connectionWith( + String product, String actualUrl, String catalog, String schema) throws SQLException { + Connection connection = mock(Connection.class); + DatabaseMetaData metadata = mock(DatabaseMetaData.class); + when(connection.getAutoCommit()).thenReturn(true); + when(connection.isReadOnly()).thenReturn(false); + when(connection.getMetaData()).thenReturn(metadata); + when(metadata.getDatabaseProductName()).thenReturn(product); + when(metadata.getURL()).thenReturn(actualUrl); + when(connection.getCatalog()).thenReturn(catalog); + when(connection.getSchema()).thenReturn(schema); + return connection; + } + + private static void awaitIgnoringInterrupt(CountDownLatch latch) { + boolean interrupted = false; + while (true) { + try { + latch.await(); + break; + } catch (InterruptedException ignored) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionLeaseTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionLeaseTest.java new file mode 100644 index 0000000000..b288eae596 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionLeaseTest.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +@Timeout(10) +class TargetJdbcConnectionLeaseTest { + + private static final String IDENTITY = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + @Test + void leaseExposesOnlyHashAndTheExactScopedConnection() { + Connection connection = mock(Connection.class); + TargetJdbcConnectionLease lease = new TargetJdbcConnectionLease(connection, IDENTITY); + + lease.withConnection(actual -> assertThat(actual).isSameAs(connection)); + + assertThat(lease.targetIdentityHash()).isEqualTo(IDENTITY); + assertThat(lease.toString()).contains(IDENTITY).doesNotContain("jdbc", "password", "user"); + } + + @Test + void nestedCallbackAndCallbackLocalCloseFailFastBeforeClosing() throws Exception { + Connection connection = mock(Connection.class); + TargetJdbcConnectionLease lease = new TargetJdbcConnectionLease(connection, IDENTITY); + + lease.withConnection(ignored -> { + assertConflict(() -> lease.withConnection(nested -> { })); + assertConflict(lease::close); + }); + + verify(connection, times(0)).close(); + lease.close(); + verify(connection).close(); + } + + @Test + void crossThreadCloseWaitsForCallbackAndClosesExactlyOnce() throws Exception { + Connection connection = mock(Connection.class); + TargetJdbcConnectionLease lease = new TargetJdbcConnectionLease(connection, IDENTITY); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch closeEntered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicReference closerThread = new AtomicReference<>(); + try (ExecutorService callers = Executors.newFixedThreadPool(2)) { + Future callback = callers.submit(() -> lease.withConnection(ignored -> { + entered.countDown(); + await(release); + })); + assertThat(entered.await(5, TimeUnit.SECONDS)).isTrue(); + Future close = callers.submit(() -> { + closerThread.set(Thread.currentThread()); + closeEntered.countDown(); + lease.close(); + }); + try { + assertThat(closeEntered.await(5, TimeUnit.SECONDS)).isTrue(); + awaitState(closerThread.get(), Thread.State.WAITING); + assertThat(close.isDone()).isFalse(); + verify(connection, times(0)).close(); + } finally { + release.countDown(); + } + callback.get(5, TimeUnit.SECONDS); + close.get(5, TimeUnit.SECONDS); + } + + lease.close(); + verify(connection).close(); + } + + @Test + void closeFailureIsSafeAndRetryableWhileFatalErrorRetainsPriority() throws Exception { + Connection connection = mock(Connection.class); + AtomicInteger closes = new AtomicInteger(); + SQLException unavailable = new SQLException("jdbc:postgresql://private/password"); + org.mockito.Mockito.doAnswer(invocation -> { + if (closes.getAndIncrement() == 0) { + throw unavailable; + } + return null; + }).when(connection).close(); + TargetJdbcConnectionLease lease = new TargetJdbcConnectionLease(connection, IDENTITY); + + assertThatThrownBy(lease::close) + .isInstanceOfSatisfying(TargetJdbcConnectionException.class, failure -> { + assertThat(failure.code()).isEqualTo(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED); + assertThat(failure).hasNoCause(); + assertThat(failure.getMessage()).doesNotContain("private", "password"); + }); + lease.close(); + verify(connection, times(2)).close(); + + Connection fatalConnection = mock(Connection.class); + AssertionError fatal = new AssertionError("private cleanup diagnostic"); + org.mockito.Mockito.doThrow(fatal).doNothing().when(fatalConnection).close(); + TargetJdbcConnectionLease fatalLease = new TargetJdbcConnectionLease(fatalConnection, IDENTITY); + assertThatThrownBy(fatalLease::close).isSameAs(fatal); + fatalLease.close(); + verify(fatalConnection, times(2)).close(); + } + + @Test + void normalCloseClearsInterruptDuringDriverCleanupAndRestoresItAfterward() throws Exception { + Connection connection = mock(Connection.class); + org.mockito.Mockito.doAnswer(invocation -> { + assertThat(Thread.currentThread().isInterrupted()).isFalse(); + return null; + }).when(connection).close(); + TargetJdbcConnectionLease lease = new TargetJdbcConnectionLease(connection, IDENTITY); + + Thread.currentThread().interrupt(); + try { + lease.close(); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } finally { + Thread.interrupted(); + } + } + + @Test + void retryableCloseRestoresInterruptAfterEveryAttempt() throws Exception { + Connection connection = mock(Connection.class); + AtomicInteger closes = new AtomicInteger(); + org.mockito.Mockito.doAnswer(invocation -> { + assertThat(Thread.currentThread().isInterrupted()).isFalse(); + if (closes.getAndIncrement() == 0) { + throw new SQLException("private cleanup diagnostic"); + } + return null; + }).when(connection).close(); + TargetJdbcConnectionLease lease = new TargetJdbcConnectionLease(connection, IDENTITY); + + Thread.currentThread().interrupt(); + try { + assertThatThrownBy(lease::close).isInstanceOf(TargetJdbcConnectionException.class); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + Thread.interrupted(); + Thread.currentThread().interrupt(); + lease.close(); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } finally { + Thread.interrupted(); + } + } + + private static void assertConflict(Runnable action) { + assertThatThrownBy(action::run) + .isInstanceOfSatisfying(TargetJdbcConnectionException.class, failure -> + assertThat(failure.code()).isEqualTo(TargetJdbcConnectionErrorCode.OPERATION_CONFLICT)); + } + + private static void await(CountDownLatch release) { + boolean interrupted = false; + while (true) { + try { + release.await(); + break; + } catch (InterruptedException ignored) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + private static void awaitState(Thread thread, Thread.State expected) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (thread.getState() != expected && System.nanoTime() - deadline < 0) { + Thread.onSpinWait(); + } + assertThat(thread.getState()).isEqualTo(expected); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionVerifierTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionVerifierTest.java new file mode 100644 index 0000000000..c1457c0c1c --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionVerifierTest.java @@ -0,0 +1,304 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.intThat; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.time.Duration; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +class TargetJdbcConnectionVerifierTest { + + private static final Duration TIMEOUT = Duration.ofSeconds(5); + private static final Executor NETWORK_EXECUTOR = Runnable::run; + + @Test + void verifiesMysqlEndpointCatalogAndStateWithoutReadingSchema() throws Exception { + AtomicLong ticker = new AtomicLong(); + Connection connection = mysqlConnection(); + TargetJdbcUrl configured = TargetJdbcUrl.parse( + MetadataDatabaseKind.MYSQL, "jdbc:mysql://db.example/hertzbeat?sslmode=required"); + + TargetJdbcConnectionLease lease = verifier().verify( + connection, configured, "operator", deadline(ticker)); + + assertThat(lease.targetIdentityHash()).matches("[0-9a-f]{64}"); + verify(connection).setNetworkTimeout(any(), intThat(value -> value > 0 && value <= 5_000)); + verify(connection, never()).getSchema(); + verify(connection, never()).close(); + lease.close(); + } + + @Test + void verifiesPostgresEndpointCatalogAndNonemptySchema() throws Exception { + AtomicLong ticker = new AtomicLong(); + Connection connection = postgresConnection(); + TargetJdbcUrl configured = TargetJdbcUrl.parse( + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db.example/hertzbeat?sslmode=require"); + + TargetJdbcConnectionLease lease = verifier().verify( + connection, configured, "operator", deadline(ticker)); + + assertThat(lease.targetIdentityHash()).isEqualTo(TargetJdbcIdentity.hash( + configured, "operator", "hertzbeat", "public")); + verify(connection).getSchema(); + lease.close(); + } + + @Test + void actualMetadataUrlCorroboratesEndpointWithoutRequiringConfiguredQuery() throws Exception { + Connection withoutQuery = connectionWith( + "PostgreSQL", "jdbc:postgresql://db.example:5432/hertzbeat", "hertzbeat", "public"); + TargetJdbcUrl configured = TargetJdbcUrl.parse( + MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://db.example/hertzbeat?sslmode=require&ApplicationName=HertzBeat"); + + TargetJdbcConnectionLease first = verifier().verify( + withoutQuery, configured, "operator", + JdbcMetadataMigrationDeadline.start(TIMEOUT, System::nanoTime)); + first.close(); + + Connection driverQuery = connectionWith( + "PostgreSQL", + "jdbc:postgresql://db.example/hertzbeat?user=operator&driverProperty=value", + "hertzbeat", "public"); + TargetJdbcConnectionLease second = verifier().verify( + driverQuery, configured, "operator", + JdbcMetadataMigrationDeadline.start(TIMEOUT, System::nanoTime)); + second.close(); + } + + @Test + void actualMetadataEndpointMismatchIsRejectedWithoutLeakingTheActualUrl() throws Exception { + assertMismatch(connectionWith( + "PostgreSQL", "jdbc:postgresql://other.example/hertzbeat?user=private", + "hertzbeat", "public"), + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db.example/hertzbeat"); + assertMismatch(connectionWith( + "PostgreSQL", "jdbc:postgresql://db.example:6432/hertzbeat", + "hertzbeat", "public"), + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db.example/hertzbeat"); + assertMismatch(connectionWith( + "PostgreSQL", "jdbc:postgresql://db.example/other", + "hertzbeat", "public"), + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db.example/hertzbeat"); + assertMismatch(connectionWith( + "PostgreSQL", "jdbc:postgresql://db-a.example,db-b.example/hertzbeat", + "hertzbeat", "public"), + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db.example/hertzbeat"); + } + + @Test + void productEndpointCatalogAndSchemaMismatchFailClosed() throws Exception { + assertMismatch(connectionWith("MariaDB", "jdbc:mysql://db.example/hertzbeat", "hertzbeat", null), + MetadataDatabaseKind.MYSQL, "jdbc:mysql://db.example/hertzbeat"); + assertMismatch(connectionWith("MySQL", "jdbc:mysql://other.example/hertzbeat", "hertzbeat", null), + MetadataDatabaseKind.MYSQL, "jdbc:mysql://db.example/hertzbeat"); + assertMismatch(connectionWith("MySQL", "jdbc:mysql://db.example/hertzbeat", "other", null), + MetadataDatabaseKind.MYSQL, "jdbc:mysql://db.example/hertzbeat"); + assertMismatch(connectionWith("PostgreSQL", "jdbc:postgresql://db.example/hertzbeat", "hertzbeat", ""), + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db.example/hertzbeat"); + } + + @ParameterizedTest + @EnumSource(ExpirationPoint.class) + void everyJdbcBoundaryUsesTheSameDeadlineAndClosesTheProvisionalConnection( + ExpirationPoint expirationPoint) throws Exception { + AtomicLong ticker = new AtomicLong(); + MetadataDatabaseKind kind = expirationPoint == ExpirationPoint.SCHEMA + ? MetadataDatabaseKind.POSTGRESQL : MetadataDatabaseKind.MYSQL; + Connection connection = expiringConnection(kind, expirationPoint, ticker); + doAnswer(invocation -> { + return null; + }).when(connection).close(); + TargetJdbcUrl configured = TargetJdbcUrl.parse(kind, kind == MetadataDatabaseKind.MYSQL + ? "jdbc:mysql://db.example/hertzbeat?sslmode=required" + : "jdbc:postgresql://db.example/hertzbeat?sslmode=require"); + + assertThatThrownBy(() -> verifier().verify(connection, configured, "operator", deadline(ticker))) + .isInstanceOfSatisfying(TargetJdbcConnectionException.class, failure -> { + assertThat(failure.code()).isEqualTo(TargetJdbcConnectionErrorCode.TIMEOUT); + assertThat(failure).hasNoCause(); + }); + + verify(connection).close(); + } + + @Test + void timeoutCleanupClearsInterruptDuringCloseAndRestoresItAfterward() throws Exception { + Connection connection = mysqlConnection(); + doAnswer(invocation -> { + assertThat(Thread.currentThread().isInterrupted()).isFalse(); + return null; + }).when(connection).close(); + TargetJdbcUrl configured = TargetJdbcUrl.parse( + MetadataDatabaseKind.MYSQL, "jdbc:mysql://db.example/hertzbeat?sslmode=required"); + + Thread.currentThread().interrupt(); + try { + assertThatThrownBy(() -> verifier().verify( + connection, configured, "operator", + JdbcMetadataMigrationDeadline.start(TIMEOUT, System::nanoTime))) + .isInstanceOfSatisfying(TargetJdbcConnectionException.class, failure -> + assertThat(failure.code()).isEqualTo(TargetJdbcConnectionErrorCode.TIMEOUT)); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } finally { + Thread.interrupted(); + } + } + + @Test + void fatalCleanupClearsNewInterruptDuringCloseAndRestoresItAfterward() throws Exception { + Connection connection = mysqlConnection(); + AssertionError fatal = new AssertionError("fatal provisional inspection"); + when(connection.getAutoCommit()).thenAnswer(invocation -> { + Thread.currentThread().interrupt(); + throw fatal; + }); + doAnswer(invocation -> { + assertThat(Thread.currentThread().isInterrupted()).isFalse(); + return null; + }).when(connection).close(); + TargetJdbcUrl configured = TargetJdbcUrl.parse( + MetadataDatabaseKind.MYSQL, "jdbc:mysql://db.example/hertzbeat?sslmode=required"); + + try { + assertThatThrownBy(() -> verifier().verify( + connection, configured, "operator", + JdbcMetadataMigrationDeadline.start(TIMEOUT, System::nanoTime))) + .isSameAs(fatal); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } finally { + Thread.interrupted(); + } + } + + private static void assertMismatch( + Connection connection, MetadataDatabaseKind kind, String configuredUrl) throws Exception { + TargetJdbcUrl configured = TargetJdbcUrl.parse(kind, configuredUrl); + assertThatThrownBy(() -> verifier().verify( + connection, configured, "operator", + JdbcMetadataMigrationDeadline.start(TIMEOUT, System::nanoTime))) + .isInstanceOfSatisfying(TargetJdbcConnectionException.class, failure -> { + assertThat(failure.code()).isEqualTo(TargetJdbcConnectionErrorCode.TARGET_MISMATCH); + assertThat(failure).hasNoCause(); + assertThat(failure.getMessage()).doesNotContain("db.example", "operator", "hertzbeat"); + }); + verify(connection).close(); + } + + private static TargetJdbcConnectionVerifier verifier() { + return new TargetJdbcConnectionVerifier(NETWORK_EXECUTOR); + } + + private static JdbcMetadataMigrationDeadline deadline(AtomicLong ticker) { + return JdbcMetadataMigrationDeadline.start(TIMEOUT, ticker::get); + } + + private static Connection mysqlConnection() throws Exception { + return connectionWith("MySQL", "jdbc:mysql://db.example/hertzbeat?sslmode=required", "hertzbeat", null); + } + + private static Connection postgresConnection() throws Exception { + return connectionWith( + "PostgreSQL", "jdbc:postgresql://db.example/hertzbeat?sslmode=require", "hertzbeat", "public"); + } + + private static Connection connectionWith( + String product, String actualUrl, String catalog, String schema) throws Exception { + Connection connection = mock(Connection.class); + DatabaseMetaData metadata = mock(DatabaseMetaData.class); + when(connection.getAutoCommit()).thenReturn(true); + when(connection.isReadOnly()).thenReturn(false); + when(connection.getMetaData()).thenReturn(metadata); + when(metadata.getDatabaseProductName()).thenReturn(product); + when(metadata.getURL()).thenReturn(actualUrl); + when(connection.getCatalog()).thenReturn(catalog); + when(connection.getSchema()).thenReturn(schema); + return connection; + } + + private static Connection expiringConnection( + MetadataDatabaseKind kind, + ExpirationPoint expirationPoint, + AtomicLong ticker) throws Exception { + Connection connection = mock(Connection.class); + DatabaseMetaData metadata = mock(DatabaseMetaData.class); + when(connection.getAutoCommit()).thenAnswer(invocation -> { + expire(expirationPoint, ExpirationPoint.AUTO_COMMIT, ticker); + return true; + }); + when(connection.isReadOnly()).thenAnswer(invocation -> { + expire(expirationPoint, ExpirationPoint.READ_ONLY, ticker); + return false; + }); + doAnswer(invocation -> { + expire(expirationPoint, ExpirationPoint.NETWORK_TIMEOUT, ticker); + return null; + }).when(connection).setNetworkTimeout(any(), anyInt()); + when(connection.getMetaData()).thenAnswer(invocation -> { + expire(expirationPoint, ExpirationPoint.METADATA, ticker); + return metadata; + }); + when(metadata.getDatabaseProductName()).thenAnswer(invocation -> { + expire(expirationPoint, ExpirationPoint.PRODUCT, ticker); + return kind == MetadataDatabaseKind.MYSQL ? "MySQL" : "PostgreSQL"; + }); + when(metadata.getURL()).thenAnswer(invocation -> { + expire(expirationPoint, ExpirationPoint.URL, ticker); + return kind == MetadataDatabaseKind.MYSQL + ? "jdbc:mysql://db.example/hertzbeat?sslmode=required" + : "jdbc:postgresql://db.example/hertzbeat?sslmode=require"; + }); + when(connection.getCatalog()).thenAnswer(invocation -> { + expire(expirationPoint, ExpirationPoint.CATALOG, ticker); + return "hertzbeat"; + }); + when(connection.getSchema()).thenAnswer(invocation -> { + expire(expirationPoint, ExpirationPoint.SCHEMA, ticker); + return "public"; + }); + return connection; + } + + private static void expire( + ExpirationPoint actual, + ExpirationPoint expected, + AtomicLong ticker) { + if (actual == expected) { + ticker.set(TIMEOUT.toNanos()); + } + } + + private enum ExpirationPoint { + AUTO_COMMIT, + READ_ONLY, + NETWORK_TIMEOUT, + METADATA, + PRODUCT, + URL, + CATALOG, + SCHEMA + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcIdentityTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcIdentityTest.java new file mode 100644 index 0000000000..f851d1677e --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcIdentityTest.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.api.Test; + +class TargetJdbcIdentityTest { + + @Test + void identityIsDeterministicLowercaseSha256() { + TargetJdbcUrl target = TargetJdbcUrl.parse( + MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://db.example/hertzbeat?sslmode=require"); + + String first = TargetJdbcIdentity.hash(target, "operator", "hertzbeat", "public"); + String second = TargetJdbcIdentity.hash(target, "operator", "hertzbeat", "public"); + + assertThat(first).isEqualTo(second).matches("[0-9a-f]{64}"); + } + + @Test + void lengthFramingPreventsDelimiterAmbiguity() { + TargetJdbcUrl target = TargetJdbcUrl.parse( + MetadataDatabaseKind.MYSQL, + "jdbc:mysql://db.example/hertzbeat"); + + String first = TargetJdbcIdentity.hash(target, "ab", "c", null); + String second = TargetJdbcIdentity.hash(target, "a", "bc", null); + + assertThat(first).isNotEqualTo(second); + } + + @Test + void everyIdentityFieldParticipatesWithoutAcceptingCredentials() { + TargetJdbcUrl mysql = TargetJdbcUrl.parse( + MetadataDatabaseKind.MYSQL, + "jdbc:mysql://db.example/hertzbeat"); + String baseline = TargetJdbcIdentity.hash(mysql, "operator", "hertzbeat", null); + + assertThat(TargetJdbcIdentity.hash(mysql, "other", "hertzbeat", null)).isNotEqualTo(baseline); + assertThat(TargetJdbcIdentity.hash(mysql, "operator", "other", null)).isNotEqualTo(baseline); + assertThat(TargetJdbcIdentity.hash( + TargetJdbcUrl.parse(MetadataDatabaseKind.MYSQL, "jdbc:mysql://other.example/hertzbeat"), + "operator", "hertzbeat", null)).isNotEqualTo(baseline); + assertThat(TargetJdbcIdentity.class.getDeclaredMethods()) + .allSatisfy(method -> assertThat(method.getParameterTypes()).doesNotContain(char[].class)); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcUrlTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcUrlTest.java new file mode 100644 index 0000000000..606b73c1b2 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcUrlTest.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.stream.Stream; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class TargetJdbcUrlTest { + + @Test + void mysqlUrlIsCanonicalWithoutChangingTheOriginalConnectionInput() { + String original = "jdbc:mysql://DB.Example:3306/hertzbeat?sslMode=VERIFY_IDENTITY&connectTimeout=1000"; + + TargetJdbcUrl parsed = TargetJdbcUrl.parse(MetadataDatabaseKind.MYSQL, original); + + assertThat(parsed.host()).isEqualTo("db.example"); + assertThat(parsed.port()).isEqualTo(3306); + assertThat(parsed.database()).isEqualTo("hertzbeat"); + assertThat(parsed.canonicalUrl()).isEqualTo( + "jdbc:mysql://db.example:3306/hertzbeat?connecttimeout=1000&sslmode=VERIFY_IDENTITY"); + assertThat(original).isEqualTo( + "jdbc:mysql://DB.Example:3306/hertzbeat?sslMode=VERIFY_IDENTITY&connectTimeout=1000"); + } + + @Test + void postgresIpv6AndEncodedDatabaseHaveOneStableCanonicalForm() { + TargetJdbcUrl parsed = TargetJdbcUrl.parse( + MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://[2001:DB8::1]/hertz%62eat?ApplicationName=Hertz%20Beat"); + + assertThat(parsed.host()).isEqualTo("[2001:db8::1]"); + assertThat(parsed.port()).isEqualTo(5432); + assertThat(parsed.database()).isEqualTo("hertzbeat"); + assertThat(parsed.canonicalUrl()).isEqualTo( + "jdbc:postgresql://[2001:db8::1]:5432/hertzbeat?applicationname=Hertz%20Beat"); + } + + @Test + void queryOrderAndEquivalentPercentEncodingDoNotChangeCanonicalIdentityInput() { + TargetJdbcUrl first = TargetJdbcUrl.parse( + MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://db.example/hertzbeat?sslmode=require&ApplicationName=Hertz%20Beat"); + TargetJdbcUrl second = TargetJdbcUrl.parse( + MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://DB.EXAMPLE:5432/hertz%62eat?applicationname=Hertz%20Beat&sslmode=require"); + + assertThat(first.canonicalUrl()).isEqualTo(second.canonicalUrl()); + assertThat(first.sameTarget(second)).isTrue(); + } + + @ParameterizedTest + @MethodSource("credentialUrls") + void credentialsInAuthorityOrDecodedQueryKeysAreRejected( + MetadataDatabaseKind kind, String url) { + assertRejected(kind, url); + } + + @ParameterizedTest + @MethodSource("ambiguousUrls") + void ambiguousOrUnsupportedUrlFormsAreRejected( + MetadataDatabaseKind kind, String url) { + assertRejected(kind, url); + } + + @Test + void safeProjectionNeverContainsEndpointOrDatabase() { + TargetJdbcUrl parsed = TargetJdbcUrl.parse( + MetadataDatabaseKind.MYSQL, + "jdbc:mysql://private.example/hertzbeat?sslMode=VERIFY_IDENTITY"); + + assertThat(parsed.toString()) + .isEqualTo("TargetJdbcUrl[kind=MYSQL]") + .doesNotContain("private.example", "hertzbeat", "sslMode"); + } + + private static Stream credentialUrls() { + return Stream.of( + Arguments.of(MetadataDatabaseKind.MYSQL, + "jdbc:mysql://operator:secret@db.example/hertzbeat"), + Arguments.of(MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://db.example/hertzbeat?user=operator"), + Arguments.of(MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://db.example/hertzbeat?UsErNaMe=operator"), + Arguments.of(MetadataDatabaseKind.MYSQL, + "jdbc:mysql://db.example/hertzbeat?%75ser=operator"), + Arguments.of(MetadataDatabaseKind.MYSQL, + "jdbc:mysql://db.example/hertzbeat?pass%77ord=secret"), + Arguments.of(MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://db.example/hertzbeat?sslpassword=secret"), + Arguments.of(MetadataDatabaseKind.MYSQL, + "jdbc:mysql://db.example/hertzbeat?trustCertificateKeyStorePassword=secret"), + Arguments.of(MetadataDatabaseKind.MYSQL, + "jdbc:mysql://db.example/hertzbeat?clientCertificateKeyStorePassword=secret"), + Arguments.of(MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://db.example/hertzbeat?client%53ecret=secret"), + Arguments.of(MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://db.example/hertzbeat?accessToken=secret")); + } + + private static Stream ambiguousUrls() { + return Stream.of( + Arguments.of(MetadataDatabaseKind.MYSQL, + "jdbc:postgresql://db.example/hertzbeat"), + Arguments.of(MetadataDatabaseKind.POSTGRESQL, + "jdbc:mysql://db.example/hertzbeat"), + Arguments.of(MetadataDatabaseKind.MYSQL, + "jdbc:mysql:loadbalance://db.example/hertzbeat"), + Arguments.of(MetadataDatabaseKind.MYSQL, + "jdbc:mysql:replication://db.example/hertzbeat"), + Arguments.of(MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://db-a.example,db-b.example/hertzbeat"), + Arguments.of(MetadataDatabaseKind.MYSQL, + "jdbc:mysql:///hertzbeat"), + Arguments.of(MetadataDatabaseKind.MYSQL, + "jdbc:mysql://db.example/"), + Arguments.of(MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://db.example/hertzbeat/other"), + Arguments.of(MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://db.example:0/hertzbeat"), + Arguments.of(MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://db.example:65536/hertzbeat"), + Arguments.of(MetadataDatabaseKind.MYSQL, + "jdbc:mysql://db.example/hertzbeat?ssl=true&SSL=false"), + Arguments.of(MetadataDatabaseKind.MYSQL, + "jdbc:mysql://db.example/hertzbeat?ssl=true&%73sl=false"), + Arguments.of(MetadataDatabaseKind.MYSQL, + "jdbc:mysql://db.example/hertzbeat?bad=%ZZ"), + Arguments.of(MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://db.example/hertzbeat#fragment")); + } + + private static void assertRejected(MetadataDatabaseKind kind, String url) { + assertThatThrownBy(() -> TargetJdbcUrl.parse(kind, url)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid target JDBC URL"); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcVendorConnectorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcVendorConnectorTest.java new file mode 100644 index 0000000000..12b44eb86f --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcVendorConnectorTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.mysql.cj.jdbc.MysqlDataSource; +import java.sql.Connection; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.AtomicLong; +import javax.sql.DataSource; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.api.Test; +import org.postgresql.ds.PGSimpleDataSource; + +class TargetJdbcVendorConnectorTest { + + @Test + void providerThatConsumesTheRootBudgetCannotSendCredentials() throws Exception { + DataSource dataSource = mock(DataSource.class); + AtomicLong ticker = new AtomicLong(); + TargetJdbcVendorConnector connector = new TargetJdbcVendorConnector(settings -> { + ticker.set(21L); + return dataSource; + }); + JdbcMetadataMigrationDeadline deadline = JdbcMetadataMigrationDeadline.start( + Duration.ofNanos(20), ticker::get); + + assertThatThrownBy(() -> connector.connect( + TargetJdbcUrl.parse(MetadataDatabaseKind.MYSQL, "jdbc:mysql://db.example/hertzbeat"), + "operator", "secret".toCharArray(), deadline)) + .isInstanceOfSatisfying(TargetJdbcConnectionException.class, failure -> { + assertThat(failure.code()).isEqualTo(TargetJdbcConnectionErrorCode.TIMEOUT); + assertThat(failure).hasNoCause(); + }); + verify(dataSource, times(0)).getConnection("operator", "secret"); + } + + @Test + void mysqlUsesExactUrlMillisecondTimeoutsAndEphemeralConnectionCredential() throws Exception { + DataSource dataSource = mock(DataSource.class); + Connection connection = mock(Connection.class); + when(dataSource.getConnection("operator", "secret")).thenReturn(connection); + AtomicReference captured = new AtomicReference<>(); + TargetJdbcVendorConnector connector = new TargetJdbcVendorConnector(settings -> { + captured.set(settings); + return dataSource; + }); + String originalUrl = "jdbc:mysql://DB.Example/hertzbeat?sslMode=VERIFY_IDENTITY"; + TargetJdbcUrl target = TargetJdbcUrl.parse(MetadataDatabaseKind.MYSQL, originalUrl); + + Connection actual = connector.connect( + target, "operator", "secret".toCharArray(), deadline(1501)); + + assertThat(actual).isSameAs(connection); + assertThat(captured.get().kind()).isEqualTo(MetadataDatabaseKind.MYSQL); + assertThat(captured.get().jdbcUrl()).isEqualTo(originalUrl); + assertThat(captured.get().remaining()).isEqualTo(Duration.ofMillis(1501)); + verify(dataSource).getConnection("operator", "secret"); + } + + @Test + void postgresUsesPositiveCeilingSecondsForConnectSocketAndCancel() throws Exception { + DataSource dataSource = mock(DataSource.class); + Connection connection = mock(Connection.class); + when(dataSource.getConnection("operator", "secret")).thenReturn(connection); + AtomicReference captured = new AtomicReference<>(); + TargetJdbcVendorConnector connector = new TargetJdbcVendorConnector(settings -> { + captured.set(settings); + return dataSource; + }); + String originalUrl = "jdbc:postgresql://db.example/hertzbeat?sslmode=require"; + TargetJdbcUrl target = TargetJdbcUrl.parse(MetadataDatabaseKind.POSTGRESQL, originalUrl); + + assertThat(connector.connect( + target, "operator", "secret".toCharArray(), deadline(1501))) + .isSameAs(connection); + + assertThat(captured.get().remaining()).isEqualTo(Duration.ofMillis(1501)); + } + + @Test + void subMillisecondBudgetNeverConfiguresAnInfiniteTimeout() throws Exception { + DataSource dataSource = mock(DataSource.class); + when(dataSource.getConnection("operator", "secret")).thenReturn(mock(Connection.class)); + AtomicReference captured = new AtomicReference<>(); + TargetJdbcVendorConnector connector = new TargetJdbcVendorConnector(settings -> { + captured.set(settings); + return dataSource; + }); + + connector.connect( + TargetJdbcUrl.parse(MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://db.example/hertzbeat"), + "operator", "secret".toCharArray(), deadlineNanos(1)); + + assertThat(captured.get().remaining()).isEqualTo(Duration.ofNanos(1)); + } + + @Test + void realVendorDataSourcesOverrideUrlTimeoutsWithoutRetainingCredentials() throws Exception { + MysqlDataSource mysql = (MysqlDataSource) TargetJdbcVendorConnector.createDataSource( + new TargetJdbcDataSourceSettings(MetadataDatabaseKind.MYSQL, + "jdbc:mysql://db.example/hertzbeat?connectTimeout=999999&socketTimeout=999999", + Duration.ofMillis(1501))); + PGSimpleDataSource postgres = (PGSimpleDataSource) TargetJdbcVendorConnector.createDataSource( + new TargetJdbcDataSourceSettings(MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://db.example/hertzbeat?connectTimeout=999999&socketTimeout=999999" + + "&cancelSignalTimeout=999999", + Duration.ofMillis(1501))); + + assertThat(mysql.getConnectTimeout()).isEqualTo(1501); + assertThat(mysql.getSocketTimeout()).isEqualTo(1501); + assertThat(mysql.getUser()).isNull(); + assertThat(mysql.getPassword()).isNull(); + assertThat(postgres.getConnectTimeout()).isEqualTo(2); + assertThat(postgres.getSocketTimeout()).isEqualTo(2); + assertThat(postgres.getCancelSignalTimeout()).isEqualTo(2); + assertThat(postgres.getUser()).isNull(); + assertThat(postgres.getPassword()).isNull(); + } + + private static JdbcMetadataMigrationDeadline deadline(long millis) { + return JdbcMetadataMigrationDeadline.start(Duration.ofMillis(millis), () -> 0L); + } + + private static JdbcMetadataMigrationDeadline deadlineNanos(long nanos) { + return JdbcMetadataMigrationDeadline.start(Duration.ofNanos(nanos), () -> 0L); + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionFactoryDatabaseTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionFactoryDatabaseTest.java new file mode 100644 index 0000000000..29fa3d4989 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionFactoryDatabaseTest.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowableOfType; + +import java.sql.Connection; +import java.sql.SQLException; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.testcontainers.mysql.MySQLContainer; +import org.testcontainers.postgresql.PostgreSQLContainer; + +/** Real-database proof for bounded target JDBC connection ownership. */ +@Timeout(120) +@EnabledIfSystemProperty(named = "hertzbeat.test.database-containers", matches = "true") +class TargetJdbcConnectionFactoryDatabaseTest { + + private static final String DATABASE = "hertzbeat"; + private static final String USERNAME = "target_test_user"; + private static final String PASSWORD = "test-only-password"; + private static final String INVALID_PASSWORD = "test-only-invalid-password"; + + @Test + void acquiresAndOwnsMysqlLease() throws Exception { + try (MySQLContainer database = new MySQLContainer("mysql:8.4") + .withDatabaseName(DATABASE) + .withUsername(USERNAME) + .withPassword(PASSWORD)) { + database.start(); + verifyLeaseAndCredentialFailure( + MetadataDatabaseKind.MYSQL, + database.getJdbcUrl(), + "MySQL", + null); + } + } + + @Test + void acquiresAndOwnsPostgresqlLease() throws Exception { + try (PostgreSQLContainer database = new PostgreSQLContainer("postgres:17.6") + .withDatabaseName(DATABASE) + .withUsername(USERNAME) + .withPassword(PASSWORD)) { + database.start(); + verifyLeaseAndCredentialFailure( + MetadataDatabaseKind.POSTGRESQL, + database.getJdbcUrl(), + "PostgreSQL", + "public"); + } + } + + private static void verifyLeaseAndCredentialFailure( + MetadataDatabaseKind kind, + String jdbcUrl, + String expectedProduct, + String expectedSchema) throws Exception { + MetadataDatabaseSettings settings = new MetadataDatabaseSettings(kind, jdbcUrl, USERNAME); + TargetJdbcConnectionLease lease; + try (TargetJdbcConnectionFactory factory = new TargetJdbcConnectionFactory(Runnable::run); + SecretValue password = SecretValue.of(PASSWORD)) { + lease = factory.acquire(settings, password, deadline()); + assertThat(lease.targetIdentityHash()).matches("[0-9a-f]{64}"); + + factory.close(); + assertExactConnection(lease, expectedProduct, expectedSchema); + lease.close(); + assertThatThrownBy(() -> lease.withConnection(connection -> { })) + .isInstanceOfSatisfying(TargetJdbcConnectionException.class, failure -> + assertThat(failure.code()) + .isEqualTo(TargetJdbcConnectionErrorCode.OPERATION_CONFLICT)); + } + assertCredentialFailure(settings, jdbcUrl); + } + + private static void assertExactConnection( + TargetJdbcConnectionLease lease, + String expectedProduct, + String expectedSchema) { + AtomicReference exactConnection = new AtomicReference<>(); + lease.withConnection(connection -> { + exactConnection.set(connection); + assertMetadata(connection, expectedProduct, expectedSchema); + }); + lease.withConnection(connection -> assertThat(connection).isSameAs(exactConnection.get())); + } + + private static void assertMetadata( + Connection connection, + String expectedProduct, + String expectedSchema) { + try { + assertThat(connection.getMetaData().getDatabaseProductName()).isEqualTo(expectedProduct); + assertThat(connection.getCatalog()).isEqualTo(DATABASE); + if (expectedSchema == null) { + assertThat(connection.getSchema()).isNull(); + } else { + assertThat(connection.getSchema()).isEqualTo(expectedSchema); + } + } catch (SQLException metadataFailure) { + throw new AssertionError("Target JDBC metadata inspection failed", metadataFailure); + } + } + + private static void assertCredentialFailure( + MetadataDatabaseSettings settings, + String jdbcUrl) { + try (TargetJdbcConnectionFactory factory = new TargetJdbcConnectionFactory(Runnable::run); + SecretValue password = SecretValue.of(INVALID_PASSWORD)) { + TargetJdbcConnectionException failure = catchThrowableOfType( + TargetJdbcConnectionException.class, + () -> factory.acquire(settings, password, deadline())); + + assertThat(failure.code()).isEqualTo(TargetJdbcConnectionErrorCode.UNAVAILABLE); + assertThat(failure).hasNoCause(); + assertThat(failure.toString()) + .doesNotContain(jdbcUrl, USERNAME, PASSWORD, INVALID_PASSWORD); + } + } + + private static JdbcMetadataMigrationDeadline deadline() { + return JdbcMetadataMigrationDeadline.start(Duration.ofSeconds(30), System::nanoTime); + } +} From 6f8258cc7daa12763709b736b5320750fe7b1aba Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 12:22:17 +0800 Subject: [PATCH 48/71] Classify metadata migration restart state --- .../MigrationOperationTransitionPolicy.java | 1 - .../workflow/MigrationRestartClassifier.java | 105 +++++++++ .../FileMigrationOperationStoreTest.java | 9 +- ...igrationOperationTransitionPolicyTest.java | 15 ++ .../MigrationRestartClassifierTest.java | 215 ++++++++++++++++++ 5 files changed, 343 insertions(+), 2 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationRestartClassifier.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationRestartClassifierTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicy.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicy.java index 558a256648..5242ea22e7 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicy.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicy.java @@ -57,7 +57,6 @@ final class MigrationOperationTransitionPolicy { case ACTIVATING -> runningAt(next, MigrationStage.ACTIVATING, VerificationState.SUCCEEDED) || rollingBackAt(next, MigrationRollbackOrigin.ACTIVATION_FAILURE) || next.state() == MigrationOperationState.AWAITING_RESTART - || next.state() == MigrationOperationState.SUCCEEDED || failedWith(next, SetupErrorCode.MIGRATION_ACTIVATION_FAILED); case ROLLING_BACK -> rollbackContinues(current, next) || rollbackCompletes(current, next); default -> false; diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationRestartClassifier.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationRestartClassifier.java new file mode 100644 index 0000000000..36503d8ef9 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationRestartClassifier.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; + +/** + * Classifies durable migration state after a restart without performing recovery I/O. + * Exact candidate evidence proves candidate identity, never the copy outcome. + */ +final class MigrationRestartClassifier { + + Plan classify(MigrationOperationSnapshot snapshot, CandidateEvidence evidence) { + Objects.requireNonNull(snapshot, "snapshot"); + Objects.requireNonNull(evidence, "evidence"); + if (evidence == CandidateEvidence.INCONSISTENT + || evidence == CandidateEvidence.RECOVERY_REQUIRED) { + return Plan.RECOVERY_REQUIRED; + } + return snapshot.applyMode() == ApplyMode.MANAGED_WRITE + ? managed(snapshot, evidence) : external(snapshot, evidence); + } + + private Plan managed(MigrationOperationSnapshot snapshot, CandidateEvidence evidence) { + if (evidence == CandidateEvidence.NOT_APPLICABLE) { + return Plan.RECOVERY_REQUIRED; + } + if (snapshot.terminal()) { + return evidence == CandidateEvidence.EXACT + ? Plan.CLEANUP_TERMINAL_CANDIDATE : Plan.NONE; + } + return switch (snapshot.state()) { + case PENDING -> evidence == CandidateEvidence.EXACT + ? Plan.RESUME_PREPARATION : Plan.CREDENTIALS_REQUIRED_FOR_PREPARATION; + case RUNNING -> managedRunning(snapshot.stage(), evidence); + case READY_TO_ACTIVATE -> exact(evidence, Plan.HOLD_READY_UNDER_STARTUP_GATE); + case AWAITING_RESTART -> exact(evidence, Plan.VERIFY_RESTART_CONVERGENCE); + case AWAITING_EXTERNAL_APPLY -> Plan.RECOVERY_REQUIRED; + case SUCCEEDED, FAILED, ROLLED_BACK -> Plan.RECOVERY_REQUIRED; + }; + } + + private Plan managedRunning(MigrationStage stage, CandidateEvidence evidence) { + if (evidence != CandidateEvidence.EXACT) { + return Plan.RECOVERY_REQUIRED; + } + return switch (stage) { + case COPYING, VERIFYING -> Plan.VERIFY_COPY_OUTCOME; + case ACTIVATING -> Plan.RECOVER_ACTIVATION; + case ROLLING_BACK -> Plan.RECOVER_ROLLBACK; + default -> Plan.RECOVERY_REQUIRED; + }; + } + + private Plan external(MigrationOperationSnapshot snapshot, CandidateEvidence evidence) { + if (evidence != CandidateEvidence.NOT_APPLICABLE) { + return Plan.RECOVERY_REQUIRED; + } + if (snapshot.terminal()) { + return Plan.NONE; + } + return switch (snapshot.state()) { + case PENDING -> Plan.CREDENTIALS_REQUIRED_FOR_PREPARATION; + case RUNNING -> snapshot.stage() == MigrationStage.COPYING + || snapshot.stage() == MigrationStage.VERIFYING + ? Plan.CREDENTIALS_REQUIRED_FOR_COPY_VERIFICATION : Plan.RECOVERY_REQUIRED; + case AWAITING_EXTERNAL_APPLY, AWAITING_RESTART -> Plan.VERIFY_RESTART_CONVERGENCE; + case READY_TO_ACTIVATE -> Plan.RECOVERY_REQUIRED; + case SUCCEEDED, FAILED, ROLLED_BACK -> Plan.RECOVERY_REQUIRED; + }; + } + + private Plan exact(CandidateEvidence evidence, Plan plan) { + return evidence == CandidateEvidence.EXACT ? plan : Plan.RECOVERY_REQUIRED; + } + + enum CandidateEvidence { + NOT_APPLICABLE, + MISSING, + EXACT, + INCONSISTENT, + RECOVERY_REQUIRED + } + + enum Plan { + NONE, + CLEANUP_TERMINAL_CANDIDATE, + RESUME_PREPARATION, + CREDENTIALS_REQUIRED_FOR_PREPARATION, + CREDENTIALS_REQUIRED_FOR_COPY_VERIFICATION, + VERIFY_COPY_OUTCOME, + HOLD_READY_UNDER_STARTUP_GATE, + RECOVER_ACTIVATION, + VERIFY_RESTART_CONVERGENCE, + RECOVER_ROLLBACK, + RECOVERY_REQUIRED + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStoreTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStoreTest.java index 8f65d25283..28bf67ddb4 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStoreTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStoreTest.java @@ -406,7 +406,14 @@ class FileMigrationOperationStoreTest { VerificationState.SUCCEEDED, null, null, 1000, false, false, false, pending.targetIdentityHash(), pending.managedCandidateGeneration()); store.compareAndTransition(pending.operationId(), MigrationOperationState.READY_TO_ACTIVATE, activating); - store.compareAndTransition(pending.operationId(), MigrationOperationState.RUNNING, succeeded(pending)); + MigrationOperationSnapshot awaitingRestart = new MigrationOperationSnapshot( + pending.operationId(), MigrationOperationState.AWAITING_RESTART, pending.target(), pending.applyMode(), + MigrationStage.AWAITING_RESTART, 100, pending.createdAt(), pending.createdAt().plusSeconds(1), null, + VerificationState.SUCCEEDED, null, null, 1000, false, true, false, + pending.targetIdentityHash(), pending.managedCandidateGeneration()); + store.compareAndTransition(pending.operationId(), MigrationOperationState.RUNNING, awaitingRestart); + store.compareAndTransition( + pending.operationId(), MigrationOperationState.AWAITING_RESTART, succeeded(pending)); } private static void assertStoreError(SetupErrorCode expected, ThrowingAction action) { diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicyTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicyTest.java index 37de72b5b7..e761745c82 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicyTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicyTest.java @@ -56,6 +56,21 @@ class MigrationOperationTransitionPolicyTest { assertAllowed(restart, succeeded); } + @Test + void managedActivationMustPersistRestartBeforeSuccess() { + MigrationOperationSnapshot activating = snapshot(MigrationOperationState.RUNNING, MigrationStage.ACTIVATING, + 100, STARTED, null, VerificationState.SUCCEEDED, null, 1000, false, false, false); + MigrationOperationSnapshot restart = snapshot(MigrationOperationState.AWAITING_RESTART, + MigrationStage.AWAITING_RESTART, 100, STARTED, null, VerificationState.SUCCEEDED, + null, 1000, false, true, false); + MigrationOperationSnapshot succeeded = snapshot(MigrationOperationState.SUCCEEDED, MigrationStage.COMPLETED, + 100, STARTED, COMPLETED, VerificationState.SUCCEEDED, null, 0, false, false, false); + + assertRejected(activating, succeeded); + assertAllowed(activating, restart); + assertAllowed(restart, succeeded); + } + @Test void acceptsExternalFailedAndRolledBackExitsButTerminalStatesStayClosed() { MigrationOperationSnapshot verifying = external(MigrationOperationState.RUNNING, MigrationStage.VERIFYING, diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationRestartClassifierTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationRestartClassifierTest.java new file mode 100644 index 0000000000..6918e32f2b --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationRestartClassifierTest.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.apache.hertzbeat.manager.setup.workflow.MigrationRestartClassifier.CandidateEvidence.EXACT; +import static org.apache.hertzbeat.manager.setup.workflow.MigrationRestartClassifier.Plan.CLEANUP_TERMINAL_CANDIDATE; +import static org.apache.hertzbeat.manager.setup.workflow.MigrationRestartClassifier.Plan.CREDENTIALS_REQUIRED_FOR_COPY_VERIFICATION; +import static org.apache.hertzbeat.manager.setup.workflow.MigrationRestartClassifier.Plan.CREDENTIALS_REQUIRED_FOR_PREPARATION; +import static org.apache.hertzbeat.manager.setup.workflow.MigrationRestartClassifier.Plan.HOLD_READY_UNDER_STARTUP_GATE; +import static org.apache.hertzbeat.manager.setup.workflow.MigrationRestartClassifier.Plan.NONE; +import static org.apache.hertzbeat.manager.setup.workflow.MigrationRestartClassifier.Plan.RECOVER_ACTIVATION; +import static org.apache.hertzbeat.manager.setup.workflow.MigrationRestartClassifier.Plan.RECOVER_ROLLBACK; +import static org.apache.hertzbeat.manager.setup.workflow.MigrationRestartClassifier.Plan.RESUME_PREPARATION; +import static org.apache.hertzbeat.manager.setup.workflow.MigrationRestartClassifier.Plan.VERIFY_COPY_OUTCOME; +import static org.apache.hertzbeat.manager.setup.workflow.MigrationRestartClassifier.Plan.VERIFY_RESTART_CONVERGENCE; + +import java.time.Instant; +import java.util.Arrays; +import java.util.Locale; +import java.util.Set; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.workflow.MigrationRestartClassifier.CandidateEvidence; +import org.apache.hertzbeat.manager.setup.workflow.MigrationRestartClassifier.Plan; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class MigrationRestartClassifierTest { + + private static final String TARGET_IDENTITY_HASH = + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + private static final String CANDIDATE_GENERATION = "candidate-generation-1"; + private static final Instant CREATED = Instant.parse("2026-08-10T01:00:00Z"); + private static final Instant STARTED = CREATED.plusSeconds(1); + private static final Instant COMPLETED = STARTED.plusSeconds(1); + private static final Plan R = Plan.RECOVERY_REQUIRED; + private final MigrationRestartClassifier classifier = new MigrationRestartClassifier(); + + @ParameterizedTest(name = "{0}-{1}-{2}-{3}") + @MethodSource("restartCases") + void classifiesEveryDurablePhaseAndEvidence( + ApplyMode mode, MigrationOperationState state, MigrationStage stage, + CandidateEvidence evidence, Plan expected) { + assertThat(classifier.classify(snapshot(mode, state, stage, progress(stage)), evidence)) + .isEqualTo(expected); + } + + @Test + void copyProgressDoesNotChangeRecoveryPlan() { + for (int progress : new int[] {0, 10, 99}) { + MigrationOperationSnapshot snapshot = snapshot( + ApplyMode.MANAGED_WRITE, MigrationOperationState.RUNNING, MigrationStage.COPYING, progress); + assertThat(classifier.classify(snapshot, EXACT)).isEqualTo(VERIFY_COPY_OUTCOME); + } + } + + @Test + void classificationNamesExposeNoConfigurationOrDataIdentity() { + Set forbidden = Set.of("jdbc", "url", "user", "password", "table", "checksum"); + assertThat(Arrays.stream(Plan.values()).map(Enum::name).map(value -> value.toLowerCase(Locale.ROOT))) + .allSatisfy(value -> assertThat(forbidden).noneMatch(value::contains)); + assertThat(Arrays.stream(CandidateEvidence.values()).map(Enum::name) + .map(value -> value.toLowerCase(Locale.ROOT))) + .allSatisfy(value -> assertThat(forbidden).noneMatch(value::contains)); + } + + @Test + void credentialPlansCannotTurnVerificationRecoveryIntoCopyExecution() { + assertThat(CREDENTIALS_REQUIRED_FOR_COPY_VERIFICATION) + .isNotEqualTo(CREDENTIALS_REQUIRED_FOR_PREPARATION); + assertThat(CREDENTIALS_REQUIRED_FOR_COPY_VERIFICATION.name()) + .contains("VERIFICATION") + .doesNotContain("RESUME", "COPY_EXECUTION"); + } + + private static Stream restartCases() { + return Stream.concat(managedCases(), externalCases()); + } + + private static Stream managedCases() { + return Stream.of( + phase(ApplyMode.MANAGED_WRITE, MigrationOperationState.PENDING, MigrationStage.QUEUED, + R, CREDENTIALS_REQUIRED_FOR_PREPARATION, RESUME_PREPARATION, R, R), + phase(ApplyMode.MANAGED_WRITE, MigrationOperationState.RUNNING, MigrationStage.COPYING, + R, R, VERIFY_COPY_OUTCOME, R, R), + phase(ApplyMode.MANAGED_WRITE, MigrationOperationState.RUNNING, MigrationStage.VERIFYING, + R, R, VERIFY_COPY_OUTCOME, R, R), + phase(ApplyMode.MANAGED_WRITE, MigrationOperationState.READY_TO_ACTIVATE, + MigrationStage.READY_TO_ACTIVATE, R, R, HOLD_READY_UNDER_STARTUP_GATE, R, R), + phase(ApplyMode.MANAGED_WRITE, MigrationOperationState.RUNNING, MigrationStage.ACTIVATING, + R, R, RECOVER_ACTIVATION, R, R), + phase(ApplyMode.MANAGED_WRITE, MigrationOperationState.AWAITING_RESTART, + MigrationStage.AWAITING_RESTART, R, R, VERIFY_RESTART_CONVERGENCE, R, R), + phase(ApplyMode.MANAGED_WRITE, MigrationOperationState.RUNNING, MigrationStage.ROLLING_BACK, + R, R, RECOVER_ROLLBACK, R, R), + phase(ApplyMode.MANAGED_WRITE, MigrationOperationState.AWAITING_EXTERNAL_APPLY, + MigrationStage.AWAITING_EXTERNAL_APPLY, R, R, R, R, R), + terminal(ApplyMode.MANAGED_WRITE, MigrationOperationState.SUCCEEDED, MigrationStage.COMPLETED, + R, NONE, CLEANUP_TERMINAL_CANDIDATE, R, R), + terminal(ApplyMode.MANAGED_WRITE, MigrationOperationState.FAILED, MigrationStage.FAILED, + R, NONE, CLEANUP_TERMINAL_CANDIDATE, R, R), + terminal(ApplyMode.MANAGED_WRITE, MigrationOperationState.ROLLED_BACK, + MigrationStage.ROLLED_BACK, R, NONE, CLEANUP_TERMINAL_CANDIDATE, R, R)) + .flatMap(stream -> stream); + } + + private static Stream externalCases() { + return Stream.of( + phase(ApplyMode.EXTERNAL_APPLY, MigrationOperationState.PENDING, MigrationStage.QUEUED, + CREDENTIALS_REQUIRED_FOR_PREPARATION, R, R, R, R), + phase(ApplyMode.EXTERNAL_APPLY, MigrationOperationState.RUNNING, MigrationStage.COPYING, + CREDENTIALS_REQUIRED_FOR_COPY_VERIFICATION, R, R, R, R), + phase(ApplyMode.EXTERNAL_APPLY, MigrationOperationState.RUNNING, MigrationStage.VERIFYING, + CREDENTIALS_REQUIRED_FOR_COPY_VERIFICATION, R, R, R, R), + phase(ApplyMode.EXTERNAL_APPLY, MigrationOperationState.AWAITING_EXTERNAL_APPLY, + MigrationStage.AWAITING_EXTERNAL_APPLY, VERIFY_RESTART_CONVERGENCE, R, R, R, R), + phase(ApplyMode.EXTERNAL_APPLY, MigrationOperationState.AWAITING_RESTART, + MigrationStage.AWAITING_RESTART, VERIFY_RESTART_CONVERGENCE, R, R, R, R), + phase(ApplyMode.EXTERNAL_APPLY, MigrationOperationState.READY_TO_ACTIVATE, + MigrationStage.READY_TO_ACTIVATE, R, R, R, R, R), + phase(ApplyMode.EXTERNAL_APPLY, MigrationOperationState.RUNNING, MigrationStage.ACTIVATING, + R, R, R, R, R), + phase(ApplyMode.EXTERNAL_APPLY, MigrationOperationState.RUNNING, MigrationStage.ROLLING_BACK, + R, R, R, R, R), + terminal(ApplyMode.EXTERNAL_APPLY, MigrationOperationState.SUCCEEDED, MigrationStage.COMPLETED, + NONE, R, R, R, R), + terminal(ApplyMode.EXTERNAL_APPLY, MigrationOperationState.FAILED, MigrationStage.FAILED, + NONE, R, R, R, R), + terminal(ApplyMode.EXTERNAL_APPLY, MigrationOperationState.ROLLED_BACK, + MigrationStage.ROLLED_BACK, NONE, R, R, R, R)) + .flatMap(stream -> stream); + } + + private static Stream phase( + ApplyMode mode, MigrationOperationState state, MigrationStage stage, Plan... expected) { + CandidateEvidence[] evidence = CandidateEvidence.values(); + return IntStream.range(0, evidence.length) + .mapToObj(index -> Arguments.of(mode, state, stage, evidence[index], expected[index])); + } + + private static Stream terminal( + ApplyMode mode, MigrationOperationState state, MigrationStage stage, Plan... expected) { + return phase(mode, state, stage, expected); + } + + private static MigrationOperationSnapshot snapshot( + ApplyMode mode, MigrationOperationState state, MigrationStage stage, int progress) { + VerificationState verification = verification(stage); + SetupErrorCode error = error(state); + MigrationRollbackOrigin rollback = rollback(state, stage); + return new MigrationOperationSnapshot( + "migration-1", state, MigrationTarget.MYSQL, mode, stage, progress, CREATED, + state == MigrationOperationState.PENDING ? null : STARTED, + terminal(state) ? COMPLETED : null, verification, error, rollback, + polling(state), state == MigrationOperationState.READY_TO_ACTIVATE, + state == MigrationOperationState.AWAITING_RESTART, + state == MigrationOperationState.AWAITING_EXTERNAL_APPLY, + TARGET_IDENTITY_HASH, mode == ApplyMode.MANAGED_WRITE ? CANDIDATE_GENERATION : null); + } + + private static int progress(MigrationStage stage) { + return switch (stage) { + case QUEUED -> 0; + case COPYING, FAILED -> 10; + default -> 100; + }; + } + + private static VerificationState verification(MigrationStage stage) { + return switch (stage) { + case QUEUED, COPYING, FAILED -> VerificationState.PENDING; + case VERIFYING -> VerificationState.RUNNING; + default -> VerificationState.SUCCEEDED; + }; + } + + private static SetupErrorCode error(MigrationOperationState state) { + return switch (state) { + case FAILED -> SetupErrorCode.MIGRATION_COPY_FAILED; + case ROLLED_BACK -> SetupErrorCode.MIGRATION_ACTIVATION_FAILED; + default -> null; + }; + } + + private static MigrationRollbackOrigin rollback( + MigrationOperationState state, MigrationStage stage) { + return state == MigrationOperationState.ROLLED_BACK || stage == MigrationStage.ROLLING_BACK + ? MigrationRollbackOrigin.ACTIVATION_FAILURE : null; + } + + private static long polling(MigrationOperationState state) { + return state == MigrationOperationState.PENDING || state == MigrationOperationState.RUNNING + || state == MigrationOperationState.AWAITING_RESTART ? 1000 : 0; + } + + private static boolean terminal(MigrationOperationState state) { + return state == MigrationOperationState.SUCCEEDED + || state == MigrationOperationState.FAILED + || state == MigrationOperationState.ROLLED_BACK; + } +} From d74564cc88b486356a7cdce157e5469ac98ca22f Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 12:40:42 +0800 Subject: [PATCH 49/71] Stage managed metadata migration targets --- .../ManagedMetadataTargetCandidate.java | 69 +++ .../config/ManagedMetadataTargetStage.java | 73 +++ ...agedMigrationConfigurationTransaction.java | 43 +- ...nagedMetadataTargetStageOwnershipTest.java | 186 ++++++++ .../ManagedMetadataTargetStageTest.java | 418 ++++++++++++++++++ 5 files changed, 785 insertions(+), 4 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMetadataTargetCandidate.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMetadataTargetStage.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMetadataTargetStageOwnershipTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMetadataTargetStageTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMetadataTargetCandidate.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMetadataTargetCandidate.java new file mode 100644 index 0000000000..b5a74d11d1 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMetadataTargetCandidate.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.util.Optional; + +/** Owns construction and cleanup of an independent metadata-target candidate bundle. */ +final class ManagedMetadataTargetCandidate { + + private ManagedMetadataTargetCandidate() { + } + + static ManagedConfigurationBundle copyReplacingMetadata( + ManagedApplicationConfig source, ManagedSecrets sourceSecrets, + MetadataDatabaseSettings target, SecretValue password) { + SecretValue metadataPassword = SecretValue.copyOf(password); + Optional telemetryPassword = Optional.empty(); + Optional mailPassword = Optional.empty(); + try { + telemetryPassword = sourceSecrets.telemetryPassword().map(SecretValue::copyOf); + mailPassword = sourceSecrets.mailPassword().map(SecretValue::copyOf); + ManagedSecrets candidateSecrets = new ManagedSecrets( + metadataPassword, telemetryPassword, mailPassword); + return new ManagedConfigurationBundle( + new ManagedApplicationConfig(copy(target), copy(source.telemetryStore()), + copy(source.optional())), + candidateSecrets); + } catch (RuntimeException | Error failure) { + metadataPassword.close(); + telemetryPassword.ifPresent(SecretValue::close); + mailPassword.ifPresent(SecretValue::close); + throw failure; + } + } + + private static MetadataDatabaseSettings copy(MetadataDatabaseSettings source) { + return new MetadataDatabaseSettings(source.kind(), source.jdbcUrl(), source.username()); + } + + private static GreptimeSettings copy(GreptimeSettings source) { + GreptimeEndpoints endpoints = source.endpoints(); + return new GreptimeSettings(new GreptimeEndpoints(endpoints.grpc(), endpoints.http()), + source.database(), source.username()); + } + + private static ManagedOptionalConfiguration copy(ManagedOptionalConfiguration source) { + return new ManagedOptionalConfiguration( + source.publicAccess().map(ManagedMetadataTargetCandidate::copy), + source.retention().map(value -> new ManagedOptionalConfiguration.RetentionSettings(value.days())), + source.mail().map(ManagedMetadataTargetCandidate::copy)); + } + + private static ManagedOptionalConfiguration.PublicAccessSettings copy( + ManagedOptionalConfiguration.PublicAccessSettings source) { + return new ManagedOptionalConfiguration.PublicAccessSettings( + source.publicBaseUrl(), source.serverOtlpHttpEndpoint(), source.serverOtlpGrpcEndpoint()); + } + + private static ManagedOptionalConfiguration.MailSettings copy( + ManagedOptionalConfiguration.MailSettings source) { + return new ManagedOptionalConfiguration.MailSettings( + source.host(), source.port(), source.security(), source.username(), source.fromAddress()); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMetadataTargetStage.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMetadataTargetStage.java new file mode 100644 index 0000000000..b5642d1547 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMetadataTargetStage.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.io.IOException; +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Builds one metadata-only migration candidate from the exact active managed pair. */ +final class ManagedMetadataTargetStage { + + private final ManagedApplicationConfigStore applications; + private final ManagedSecretStore secrets; + private final CandidateStager stager; + + ManagedMetadataTargetStage(ManagedApplicationConfigStore applications, ManagedSecretStore secrets, + CandidateStager stager) { + this.applications = applications; + this.secrets = secrets; + this.stager = stager; + } + + ManagedMigrationConfigurationTransaction.MetadataTargetStageResult stage( + ManagedMigrationConfigurationTransaction.CandidateRef reference, String targetIdentityHash, + MetadataDatabaseSettings target, SecretValue password) throws IOException { + CandidateRead activeApplication = applications.readActive(); + CandidateRead activeSecrets = secrets.readActive(); + try { + if (!ManagedConfigurationTransaction.validPair(activeApplication, activeSecrets)) { + return result(ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED, reference); + } + ManagedApplicationConfig source = activeApplication.value().orElseThrow(); + ManagedSecrets sourceSecrets = activeSecrets.value().orElseThrow(); + try { + new ManagedConfigurationBundle(source, sourceSecrets); + } catch (IllegalArgumentException failure) { + return result(ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED, reference); + } + if (source.metadataDatabase().kind() != MetadataDatabaseKind.H2) { + return result(ManagedMigrationConfigurationTransaction.StageOutcome.SOURCE_UNSUPPORTED, reference); + } + String baseGeneration = activeApplication.generation().orElseThrow(); + try (ManagedConfigurationBundle candidate = ManagedMetadataTargetCandidate.copyReplacingMetadata( + source, sourceSecrets, target, password)) { + return result(stager.stage(reference, baseGeneration, targetIdentityHash, candidate), reference); + } + } finally { + ManagedConfigurationTransaction.close(activeSecrets); + } + } + + private static ManagedMigrationConfigurationTransaction.MetadataTargetStageResult result( + ManagedMigrationConfigurationTransaction.StageOutcome outcome, + ManagedMigrationConfigurationTransaction.CandidateRef reference) { + Optional candidate = + outcome == ManagedMigrationConfigurationTransaction.StageOutcome.STAGED + || outcome == ManagedMigrationConfigurationTransaction.StageOutcome.ALREADY_STAGED + ? Optional.of(reference) : Optional.empty(); + return new ManagedMigrationConfigurationTransaction.MetadataTargetStageResult(outcome, candidate); + } + + @FunctionalInterface + interface CandidateStager { + ManagedMigrationConfigurationTransaction.StageOutcome stage( + ManagedMigrationConfigurationTransaction.CandidateRef reference, String baseGeneration, + String targetIdentityHash, ManagedConfigurationBundle candidate) throws IOException; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransaction.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransaction.java index 42da189739..44ac3bdf02 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransaction.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransaction.java @@ -13,6 +13,7 @@ import java.util.Objects; import java.util.Optional; import java.util.regex.Pattern; import org.apache.hertzbeat.manager.setup.api.OperationIdValidator; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; /** Locked public boundary for migration-owned managed-configuration candidates. */ public final class ManagedMigrationConfigurationTransaction { @@ -23,14 +24,30 @@ public final class ManagedMigrationConfigurationTransaction { private final ManagedConfigurationLock lock; private final MigrationCandidateStore store; private final ManagedMigrationActivation activation; + private final ManagedMetadataTargetStage metadataTargetStage; /** Creates the production migration candidate transaction. */ public ManagedMigrationConfigurationTransaction(Path installationRoot) { lock = new ManagedConfigurationLock(installationRoot); store = new MigrationCandidateStore(installationRoot); - activation = new ManagedMigrationActivation( - new FileManagedApplicationConfigStore(installationRoot), - new FileManagedSecretStore(installationRoot)); + FileManagedApplicationConfigStore applications = new FileManagedApplicationConfigStore(installationRoot); + FileManagedSecretStore secrets = new FileManagedSecretStore(installationRoot); + activation = new ManagedMigrationActivation(applications, secrets); + metadataTargetStage = new ManagedMetadataTargetStage(applications, secrets, store::stage); + } + + /** Stages a metadata-only target over the exact active H2 managed configuration. */ + public MetadataTargetStageResult stageMetadataTarget( + String operationId, String candidateGeneration, String targetIdentityHash, + MetadataDatabaseSettings target, SecretValue password) throws IOException { + CandidateRef reference = new CandidateRef(operationId, candidateGeneration); + requireIdentityHash(targetIdentityHash); + Objects.requireNonNull(target, "target"); + Objects.requireNonNull(password, "password"); + if (target.kind() == MetadataDatabaseKind.H2) { + throw new IllegalArgumentException("A migration target must use a production metadata database"); + } + return lock.execute(() -> metadataTargetStage.stage(reference, targetIdentityHash, target, password)); } /** Stages the exact candidate or fails with a stable, secret-free error. */ @@ -120,6 +137,24 @@ public final class ManagedMigrationConfigurationTransaction { } } + /** Secret-free result of staging a metadata-only target candidate. */ + public record MetadataTargetStageResult(StageOutcome outcome, Optional candidate) { + public MetadataTargetStageResult { + Objects.requireNonNull(outcome, "outcome"); + Objects.requireNonNull(candidate, "candidate"); + boolean staged = outcome == StageOutcome.STAGED || outcome == StageOutcome.ALREADY_STAGED; + if (staged != candidate.isPresent()) { + throw new IllegalArgumentException("Only a staged metadata target exposes a candidate reference"); + } + } + + @Override + public String toString() { + return "MetadataTargetStageResult[outcome=" + outcome + + ", candidatePresent=" + candidate.isPresent() + "]"; + } + } + /** Secret-free persisted candidate state and exact base/target identity metadata. */ public record Inspection(CandidateState state, Optional baseGeneration, Optional targetIdentityHash) { @@ -143,7 +178,7 @@ public final class ManagedMigrationConfigurationTransaction { public enum CandidateState { MISSING, READY, RECOVERY_REQUIRED } /** Stable staging result without filesystem or configuration details. */ - public enum StageOutcome { STAGED, ALREADY_STAGED, STALE, RECOVERY_REQUIRED } + public enum StageOutcome { STAGED, ALREADY_STAGED, STALE, SOURCE_UNSUPPORTED, RECOVERY_REQUIRED } /** Stable exact-discard result. */ public enum DiscardOutcome { DISCARDED, NOT_FOUND } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMetadataTargetStageOwnershipTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMetadataTargetStageOwnershipTest.java new file mode 100644 index 0000000000..d5bda993ae --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMetadataTargetStageOwnershipTest.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +class ManagedMetadataTargetStageOwnershipTest { + + private static final ManagedMigrationConfigurationTransaction.CandidateRef REFERENCE = + new ManagedMigrationConfigurationTransaction.CandidateRef("metadata-migration", "target-generation"); + private static final String IDENTITY = + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + @AfterEach + void clearInterrupt() { + Thread.interrupted(); + } + + @ParameterizedTest + @MethodSource("storeOutcomes") + void clearsEveryOwnedSecretForEveryStoreOutcome( + ManagedMigrationConfigurationTransaction.StageOutcome storeOutcome) throws Exception { + Fixture fixture = fixture(MetadataDatabaseKind.H2); + MetadataDatabaseSettings target = target(); + AtomicReference candidate = new AtomicReference<>(); + ManagedMetadataTargetStage stage = new ManagedMetadataTargetStage( + fixture.applications(), fixture.secretStore(), (reference, base, identity, bundle) -> { + candidate.set(bundle); + assertNotSame(target, bundle.application().metadataDatabase()); + assertNotSame(fixture.application().telemetryStore(), bundle.application().telemetryStore()); + assertNotSame(fixture.application().optional(), bundle.application().optional()); + assertNotSame(fixture.application().optional().publicAccess().orElseThrow(), + bundle.application().optional().publicAccess().orElseThrow()); + assertNotSame(fixture.application().optional().retention().orElseThrow(), + bundle.application().optional().retention().orElseThrow()); + assertNotSame(fixture.application().optional().mail().orElseThrow(), + bundle.application().optional().mail().orElseThrow()); + assertEquals("telemetry-password", + new String(bundle.secrets().telemetryPassword().orElseThrow().copy())); + assertEquals("mail-password", + new String(bundle.secrets().mailPassword().orElseThrow().copy())); + return storeOutcome; + }); + + try (SecretValue borrowed = SecretValue.of("target-password")) { + ManagedMigrationConfigurationTransaction.MetadataTargetStageResult result = + stage.stage(REFERENCE, IDENTITY, target, borrowed); + + assertEquals(storeOutcome, result.outcome()); + assertEquals("target-password", new String(borrowed.copy())); + } + assertCleared(fixture.sourceSecrets().metadataDatabasePassword()); + assertCleared(fixture.sourceSecrets().telemetryPassword().orElseThrow()); + assertCleared(fixture.sourceSecrets().mailPassword().orElseThrow()); + assertCleared(candidate.get().secrets().metadataDatabasePassword()); + assertCleared(candidate.get().secrets().telemetryPassword().orElseThrow()); + assertCleared(candidate.get().secrets().mailPassword().orElseThrow()); + } + + @Test + void clearsOwnedSecretsWhenPersistenceThrowsUncheckedFailure() { + Fixture fixture = fixture(MetadataDatabaseKind.H2); + AtomicReference candidate = new AtomicReference<>(); + ManagedMetadataTargetStage stage = new ManagedMetadataTargetStage( + fixture.applications(), fixture.secretStore(), (reference, base, identity, bundle) -> { + candidate.set(bundle); + throw new IllegalStateException("stop"); + }); + + try (SecretValue borrowed = SecretValue.of("target-password")) { + assertThrows(IllegalStateException.class, + () -> stage.stage(REFERENCE, IDENTITY, target(), borrowed)); + assertEquals("target-password", new String(borrowed.copy())); + } + assertCleared(fixture.sourceSecrets().metadataDatabasePassword()); + assertCleared(candidate.get().secrets().metadataDatabasePassword()); + } + + @Test + void clearsOwnedSecretsWhenPersistenceThrowsFatalFailure() { + Fixture fixture = fixture(MetadataDatabaseKind.H2); + AtomicReference candidate = new AtomicReference<>(); + ManagedMetadataTargetStage stage = new ManagedMetadataTargetStage( + fixture.applications(), fixture.secretStore(), (reference, base, identity, bundle) -> { + candidate.set(bundle); + throw new AssertionError("stop"); + }); + + try (SecretValue borrowed = SecretValue.of("target-password")) { + assertThrows(AssertionError.class, + () -> stage.stage(REFERENCE, IDENTITY, target(), borrowed)); + assertEquals("target-password", new String(borrowed.copy())); + } + assertCleared(fixture.sourceSecrets().metadataDatabasePassword()); + assertCleared(candidate.get().secrets().metadataDatabasePassword()); + } + + @Test + void clearsOwnedSecretsAndPreservesInterruptWhenPersistenceFails() { + Fixture fixture = fixture(MetadataDatabaseKind.H2); + AtomicReference candidate = new AtomicReference<>(); + ManagedMetadataTargetStage stage = new ManagedMetadataTargetStage( + fixture.applications(), fixture.secretStore(), (reference, base, identity, bundle) -> { + candidate.set(bundle); + Thread.currentThread().interrupt(); + throw new IOException("write failed"); + }); + + try (SecretValue borrowed = SecretValue.of("target-password")) { + assertThrows(IOException.class, () -> stage.stage(REFERENCE, IDENTITY, target(), borrowed)); + assertEquals("target-password", new String(borrowed.copy())); + assertTrue(Thread.currentThread().isInterrupted()); + } + assertCleared(fixture.sourceSecrets().metadataDatabasePassword()); + assertCleared(candidate.get().secrets().metadataDatabasePassword()); + } + + private static Stream storeOutcomes() { + return Stream.of( + ManagedMigrationConfigurationTransaction.StageOutcome.STAGED, + ManagedMigrationConfigurationTransaction.StageOutcome.ALREADY_STAGED, + ManagedMigrationConfigurationTransaction.StageOutcome.STALE, + ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED); + } + + private static Fixture fixture(MetadataDatabaseKind kind) { + ManagedApplicationConfigStore applications = mock(ManagedApplicationConfigStore.class); + ManagedSecretStore secrets = mock(ManagedSecretStore.class); + ManagedApplicationConfig application = application(kind); + ManagedSecrets sourceSecrets = new ManagedSecrets(SecretValue.of("source-password"), + Optional.of(SecretValue.of("telemetry-password")), + Optional.of(SecretValue.of("mail-password"))); + when(applications.readActive()).thenReturn(CandidateRead.valid(application, "base-generation")); + when(secrets.readActive()).thenReturn(CandidateRead.valid(sourceSecrets, "base-generation")); + return new Fixture(applications, secrets, application, sourceSecrets); + } + + private static ManagedApplicationConfig application(MetadataDatabaseKind kind) { + ManagedOptionalConfiguration.PublicAccessSettings publicAccess = + new ManagedOptionalConfiguration.PublicAccessSettings( + Optional.of("https://monitor.example"), Optional.of("https://monitor.example/api/otlp"), + Optional.of("https://monitor.example:4317")); + ManagedOptionalConfiguration.MailSettings mail = new ManagedOptionalConfiguration.MailSettings( + "smtp.example", 587, MailSecurity.STARTTLS, Optional.of("mailer"), "alerts@example.org"); + return new ManagedApplicationConfig( + new MetadataDatabaseSettings(kind, "jdbc:h2:file:./data/hertzbeat", "source-user"), + new GreptimeSettings(new GreptimeEndpoints("greptime:4001", "http://greptime:4000"), + "public", Optional.of("telemetry-user")), + new ManagedOptionalConfiguration(Optional.of(publicAccess), + Optional.of(new ManagedOptionalConfiguration.RetentionSettings(30)), Optional.of(mail))); + } + + private static MetadataDatabaseSettings target() { + return new MetadataDatabaseSettings( + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db.example/hertzbeat", "target-user"); + } + + private static void assertCleared(SecretValue secret) { + assertTrue(new String(secret.copy()).chars().allMatch(value -> value == 0)); + } + + private record Fixture(ManagedApplicationConfigStore applications, ManagedSecretStore secretStore, + ManagedApplicationConfig application, ManagedSecrets sourceSecrets) { + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMetadataTargetStageTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMetadataTargetStageTest.java new file mode 100644 index 0000000000..dc0c1c57ba --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMetadataTargetStageTest.java @@ -0,0 +1,418 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFileLock; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class ManagedMetadataTargetStageTest { + + private static final String OPERATION = "metadata-migration"; + private static final String CANDIDATE = "target-generation"; + private static final String IDENTITY = + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + @TempDir + private Path installationRoot; + + @Test + void stagesOnlyTheMetadataTargetAndPreservesEveryOtherManagedSetting() throws Exception { + try (ManagedConfigurationBundle source = sourceBundle(MetadataDatabaseKind.H2)) { + assertEquals(ManagedConfigurationTransaction.Outcome.APPLIED, + new ManagedConfigurationTransaction(installationRoot).apply(source)); + } + Map before = managedSnapshotBytes(); + ManagedMigrationConfigurationTransaction transaction = + new ManagedMigrationConfigurationTransaction(installationRoot); + MetadataDatabaseSettings target = new MetadataDatabaseSettings( + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db.example/hertzbeat", "target-user"); + + try (SecretValue borrowed = SecretValue.of("target-password")) { + ManagedMigrationConfigurationTransaction.MetadataTargetStageResult result = + transaction.stageMetadataTarget(OPERATION, CANDIDATE, IDENTITY, target, borrowed); + + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.STAGED, result.outcome()); + assertEquals(new ManagedMigrationConfigurationTransaction.CandidateRef(OPERATION, CANDIDATE), + result.candidate().orElseThrow()); + assertEquals("target-password", new String(borrowed.copy())); + transaction.readExact(result.candidate().orElseThrow(), candidate -> { + assertEquals(target, candidate.application().metadataDatabase()); + assertEquals(sourceApplication(MetadataDatabaseKind.H2).telemetryStore(), + candidate.application().telemetryStore()); + assertEquals(sourceApplication(MetadataDatabaseKind.H2).optional(), + candidate.application().optional()); + assertEquals("target-password", + new String(candidate.secrets().metadataDatabasePassword().copy())); + assertEquals("telemetry-password", + new String(candidate.secrets().telemetryPassword().orElseThrow().copy())); + assertEquals("mail-password", + new String(candidate.secrets().mailPassword().orElseThrow().copy())); + return null; + }); + + Map candidateBeforeRetry = candidateBytes(CANDIDATE); + FileTime fixedTime = FileTime.fromMillis(1_000_000); + for (Path path : candidateBeforeRetry.keySet()) { + Files.setLastModifiedTime(path, fixedTime); + } + ManagedMigrationConfigurationTransaction.MetadataTargetStageResult repeated = + transaction.stageMetadataTarget(OPERATION, CANDIDATE, IDENTITY, target, borrowed); + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.ALREADY_STAGED, + repeated.outcome()); + assertEquals(result.candidate(), repeated.candidate()); + assertCandidateBytesUnchanged(candidateBeforeRetry); + for (Path path : candidateBeforeRetry.keySet()) { + assertEquals(fixedTime, Files.getLastModifiedTime(path)); + } + } + + assertManagedSnapshotsUnchanged(before); + } + + @Test + void refusesUnsupportedSourceAndInvalidActivePairsWithoutCreatingCandidate() throws Exception { + ManagedMigrationConfigurationTransaction transaction = + new ManagedMigrationConfigurationTransaction(installationRoot); + MetadataDatabaseSettings target = new MetadataDatabaseSettings( + MetadataDatabaseKind.MYSQL, "jdbc:mysql://db.example/hertzbeat", "target-user"); + try (ManagedConfigurationBundle source = sourceBundle(MetadataDatabaseKind.POSTGRESQL); + SecretValue borrowed = SecretValue.of("target-password")) { + new ManagedConfigurationTransaction(installationRoot).apply(source); + ManagedMigrationConfigurationTransaction.MetadataTargetStageResult stale = + transaction.stageMetadataTarget(OPERATION, CANDIDATE, IDENTITY, target, borrowed); + + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.SOURCE_UNSUPPORTED, + stale.outcome()); + assertTrue(stale.candidate().isEmpty()); + assertEquals("target-password", new String(borrowed.copy())); + } + + Files.delete(installationRoot.resolve("data/config/managed-secrets.properties")); + try (SecretValue borrowed = SecretValue.of("target-password")) { + ManagedMigrationConfigurationTransaction.MetadataTargetStageResult recovery = + transaction.stageMetadataTarget(OPERATION, "recovery-generation", IDENTITY, target, borrowed); + + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED, + recovery.outcome()); + assertTrue(recovery.candidate().isEmpty()); + assertEquals("target-password", new String(borrowed.copy())); + } + } + + @Test + void rejectsH2TargetBeforeCreatingCandidateOrConsumingBorrowedPassword() { + ManagedMigrationConfigurationTransaction transaction = + new ManagedMigrationConfigurationTransaction(installationRoot); + MetadataDatabaseSettings target = new MetadataDatabaseSettings( + MetadataDatabaseKind.H2, "jdbc:h2:file:./data/target", "target-user"); + try (SecretValue borrowed = SecretValue.of("target-password")) { + assertThrows(IllegalArgumentException.class, () -> transaction.stageMetadataTarget( + OPERATION, CANDIDATE, IDENTITY, target, borrowed)); + assertEquals("target-password", new String(borrowed.copy())); + assertFalse(Files.exists(candidateDirectory(CANDIDATE))); + } + } + + @Test + void conflictingExactRetryNeverOverwritesTheFirstCandidate() throws Exception { + try (ManagedConfigurationBundle source = sourceBundle(MetadataDatabaseKind.H2)) { + new ManagedConfigurationTransaction(installationRoot).apply(source); + } + ManagedMigrationConfigurationTransaction transaction = + new ManagedMigrationConfigurationTransaction(installationRoot); + MetadataDatabaseSettings first = new MetadataDatabaseSettings( + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db.example/hertzbeat", "target-user"); + MetadataDatabaseSettings different = new MetadataDatabaseSettings( + MetadataDatabaseKind.MYSQL, "jdbc:mysql://other.example/hertzbeat", "other-user"); + try (SecretValue firstPassword = SecretValue.of("first-password"); + SecretValue otherPassword = SecretValue.of("other-password")) { + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.STAGED, + transaction.stageMetadataTarget( + OPERATION, CANDIDATE, IDENTITY, first, firstPassword).outcome()); + Map original = candidateBytes(CANDIDATE); + + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED, + transaction.stageMetadataTarget( + OPERATION, CANDIDATE, "f".repeat(64), first, firstPassword).outcome()); + assertCandidateBytesUnchanged(original); + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED, + transaction.stageMetadataTarget( + OPERATION, CANDIDATE, IDENTITY, different, firstPassword).outcome()); + assertCandidateBytesUnchanged(original); + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED, + transaction.stageMetadataTarget( + OPERATION, CANDIDATE, IDENTITY, first, otherPassword).outcome()); + assertCandidateBytesUnchanged(original); + assertEquals("first-password", new String(firstPassword.copy())); + assertEquals("other-password", new String(otherPassword.copy())); + } + } + + @Test + void partialExactCandidateAndLaterActiveGenerationAreNeverOverwritten() throws Exception { + try (ManagedConfigurationBundle source = sourceBundle(MetadataDatabaseKind.H2)) { + new ManagedConfigurationTransaction(installationRoot).apply(source); + } + ManagedMigrationConfigurationTransaction transaction = + new ManagedMigrationConfigurationTransaction(installationRoot); + MetadataDatabaseSettings target = new MetadataDatabaseSettings( + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db.example/hertzbeat", "target-user"); + Path partialApplication = candidateDirectory("partial-generation").resolve("application"); + byte[] partial = "partial".getBytes(StandardCharsets.UTF_8); + try { + SecureSetupFile.create(installationRoot, partialApplication, partial); + } finally { + Arrays.fill(partial, (byte) 0); + } + byte[] partialBefore = Files.readAllBytes(partialApplication); + + try (SecretValue borrowed = SecretValue.of("target-password")) { + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED, + transaction.stageMetadataTarget( + OPERATION, "partial-generation", IDENTITY, target, borrowed).outcome()); + assertArrayEquals(partialBefore, Files.readAllBytes(partialApplication)); + assertFalse(Files.exists(partialApplication.resolveSibling("secrets"))); + assertFalse(Files.exists(partialApplication.resolveSibling("manifest"))); + Path partialSecrets = partialApplication.resolveSibling("secrets"); + byte[] secretBytes = "partial-secrets".getBytes(StandardCharsets.UTF_8); + try { + SecureSetupFile.create(installationRoot, partialSecrets, secretBytes); + } finally { + Arrays.fill(secretBytes, (byte) 0); + } + byte[] partialSecretsBefore = Files.readAllBytes(partialSecrets); + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED, + transaction.stageMetadataTarget( + OPERATION, "partial-generation", IDENTITY, target, borrowed).outcome()); + assertArrayEquals(partialBefore, Files.readAllBytes(partialApplication)); + assertArrayEquals(partialSecretsBefore, Files.readAllBytes(partialSecrets)); + assertFalse(Files.exists(partialApplication.resolveSibling("manifest"))); + + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.STAGED, + transaction.stageMetadataTarget( + OPERATION, CANDIDATE, IDENTITY, target, borrowed).outcome()); + Map original = candidateBytes(CANDIDATE); + try (ManagedConfigurationBundle later = sourceBundle(MetadataDatabaseKind.POSTGRESQL)) { + new ManagedConfigurationTransaction(installationRoot).apply(later); + } + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.SOURCE_UNSUPPORTED, + transaction.stageMetadataTarget( + OPERATION, CANDIDATE, IDENTITY, target, borrowed).outcome()); + assertCandidateBytesUnchanged(original); + } + } + + @Test + void aggregateInvalidActivePairRequiresRecoveryAndLeavesBorrowedPasswordAlone() throws Exception { + try (ManagedConfigurationBundle source = sourceBundle(MetadataDatabaseKind.H2)) { + new ManagedConfigurationTransaction(installationRoot).apply(source); + } + CandidateRead active = new FileManagedSecretStore(installationRoot).readActive(); + String generation; + try { + generation = active.generation().orElseThrow(); + } finally { + ManagedConfigurationTransaction.close(active); + } + ManagedSecrets mismatched = new ManagedSecrets(SecretValue.of("source-password"), Optional.empty(), + Optional.of(SecretValue.of("mail-password"))); + byte[] encoded = new SecretConfigDocumentCodec().encode(mismatched, generation); + try { + Files.write(installationRoot.resolve("data/config/managed-secrets.properties"), encoded); + } finally { + Arrays.fill(encoded, (byte) 0); + mismatched.close(); + } + MetadataDatabaseSettings target = new MetadataDatabaseSettings( + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db.example/hertzbeat", "target-user"); + + try (SecretValue borrowed = SecretValue.of("target-password")) { + ManagedMigrationConfigurationTransaction.MetadataTargetStageResult result = + new ManagedMigrationConfigurationTransaction(installationRoot).stageMetadataTarget( + OPERATION, CANDIDATE, IDENTITY, target, borrowed); + + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED, + result.outcome()); + assertTrue(result.candidate().isEmpty()); + assertEquals("target-password", new String(borrowed.copy())); + } + } + + @Test + void resultDiagnosticsExposeNoConfigurationOrSecretMaterial() { + ManagedMigrationConfigurationTransaction.MetadataTargetStageResult result = + new ManagedMigrationConfigurationTransaction.MetadataTargetStageResult( + ManagedMigrationConfigurationTransaction.StageOutcome.STAGED, + Optional.of(new ManagedMigrationConfigurationTransaction.CandidateRef( + OPERATION, CANDIDATE))); + + String diagnostic = result.toString(); + assertFalse(diagnostic.contains("jdbc:")); + assertFalse(diagnostic.contains("target-user")); + assertFalse(diagnostic.contains("password")); + assertFalse(diagnostic.contains("base")); + } + + @Test + void setupAndMultipleMigrationTransactionsShareTheVersionedLock() throws Exception { + ManagedConfigurationTransaction setup = new ManagedConfigurationTransaction(installationRoot); + try (ManagedConfigurationBundle source = sourceBundle(MetadataDatabaseKind.H2)) { + setup.apply(source); + } + ManagedMigrationConfigurationTransaction first = + new ManagedMigrationConfigurationTransaction(installationRoot); + ManagedMigrationConfigurationTransaction second = + new ManagedMigrationConfigurationTransaction(installationRoot); + MetadataDatabaseSettings target = new MetadataDatabaseSettings( + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db.example/hertzbeat", "target-user"); + SecureSetupFileLock held = new SecureSetupFileLock( + installationRoot, "data/config/.managed-config-v2.lock"); + CountDownLatch locked = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch callersStarted = new CountDownLatch(3); + ExecutorService executor = Executors.newFixedThreadPool(4); + try (SecretValue firstPassword = SecretValue.of("first-password"); + SecretValue secondPassword = SecretValue.of("second-password")) { + Future holder = executor.submit(() -> { + held.execute(() -> { + locked.countDown(); + try { + release.await(); + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + throw new IOException("Lock holder interrupted"); + } + }); + return null; + }); + locked.await(); + Future firstStage = executor.submit(() -> { + callersStarted.countDown(); + return first.stageMetadataTarget( + OPERATION, CANDIDATE, IDENTITY, target, firstPassword); + }); + Future secondStage = executor.submit(() -> { + callersStarted.countDown(); + return second.stageMetadataTarget( + "second-operation", "second-generation", "f".repeat(64), target, secondPassword); + }); + Future setupUpdate = executor.submit(() -> { + callersStarted.countDown(); + try (ManagedConfigurationBundle update = sourceBundle(MetadataDatabaseKind.H2)) { + return setup.apply(update); + } + }); + callersStarted.await(); + + assertFalse(firstStage.isDone()); + assertFalse(secondStage.isDone()); + assertFalse(setupUpdate.isDone()); + release.countDown(); + holder.get(); + firstStage.get(); + secondStage.get(); + setupUpdate.get(); + } finally { + release.countDown(); + executor.shutdownNow(); + } + } + + private Map managedSnapshotBytes() throws Exception { + Path directory = installationRoot.resolve("data/config"); + Map snapshots = new LinkedHashMap<>(); + for (String name : new String[] { + "managed-application.yml", "managed-application.yml.candidate", + "managed-application.yml.last-known-good", "managed-secrets.properties", + "managed-secrets.properties.candidate", "managed-secrets.properties.last-known-good"}) { + Path path = directory.resolve(name); + if (Files.exists(path)) { + snapshots.put(path, Files.readAllBytes(path)); + } + } + return snapshots; + } + + private void assertManagedSnapshotsUnchanged(Map before) throws Exception { + assertEquals(before.keySet(), managedSnapshotBytes().keySet()); + for (Map.Entry entry : before.entrySet()) { + assertArrayEquals(entry.getValue(), Files.readAllBytes(entry.getKey())); + } + } + + private Map candidateBytes(String generation) throws Exception { + Map snapshots = new LinkedHashMap<>(); + for (String name : new String[] {"application", "secrets", "manifest"}) { + Path path = candidateDirectory(generation).resolve(name); + snapshots.put(path, Files.readAllBytes(path)); + } + return snapshots; + } + + private void assertCandidateBytesUnchanged(Map expected) throws Exception { + for (Map.Entry entry : expected.entrySet()) { + assertArrayEquals(entry.getValue(), Files.readAllBytes(entry.getKey())); + } + } + + private Path candidateDirectory(String generation) { + return installationRoot.resolve("data/config/migration-candidates") + .resolve(OPERATION).resolve(generation); + } + + private static ManagedConfigurationBundle sourceBundle(MetadataDatabaseKind kind) { + return new ManagedConfigurationBundle(sourceApplication(kind), + new ManagedSecrets(SecretValue.of("source-password"), + Optional.of(SecretValue.of("telemetry-password")), + Optional.of(SecretValue.of("mail-password")))); + } + + private static ManagedApplicationConfig sourceApplication(MetadataDatabaseKind kind) { + MetadataDatabaseSettings metadata = switch (kind) { + case H2 -> new MetadataDatabaseSettings(kind, "jdbc:h2:file:./data/hertzbeat", "source-user"); + case MYSQL -> new MetadataDatabaseSettings(kind, "jdbc:mysql://source/hertzbeat", "source-user"); + case POSTGRESQL -> new MetadataDatabaseSettings( + kind, "jdbc:postgresql://source/hertzbeat", "source-user"); + }; + ManagedOptionalConfiguration.PublicAccessSettings publicAccess = + new ManagedOptionalConfiguration.PublicAccessSettings( + Optional.of("https://monitor.example"), + Optional.of("https://monitor.example/api/otlp"), + Optional.of("https://monitor.example:4317")); + ManagedOptionalConfiguration.MailSettings mail = new ManagedOptionalConfiguration.MailSettings( + "smtp.example", 587, MailSecurity.STARTTLS, Optional.of("mailer"), "alerts@example.org"); + return new ManagedApplicationConfig(metadata, + new GreptimeSettings(new GreptimeEndpoints("greptime:4001", "http://greptime:4000"), + "public", Optional.of("telemetry-user")), + new ManagedOptionalConfiguration(Optional.of(publicAccess), + Optional.of(new ManagedOptionalConfiguration.RetentionSettings(30)), Optional.of(mail))); + } +} From 5046236b82e0f947695adc14e420024fd5bae9b2 Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 13:21:39 +0800 Subject: [PATCH 50/71] Retain metadata cutover maintenance --- .../workflow/RetainedCutoverCoordinator.java | 237 ++++++++++++ .../workflow/RetainedCutoverErrorCode.java | 15 + .../workflow/RetainedCutoverException.java | 27 ++ .../workflow/RetainedCutoverOutcome.java | 92 +++++ .../workflow/RetainedCutoverRelease.java | 86 +++++ ...tainedCutoverReleaseRequiredException.java | 25 ++ .../setup/workflow/RetainedCutoverResult.java | 31 ++ .../setup/workflow/RetainedCutoverState.java | 111 ++++++ .../setup/workflow/RetainedCutoverSteps.java | 82 +++++ .../workflow/TargetJdbcConnectionFactory.java | 45 +++ .../TargetJdbcFailedAcquireSettlement.java | 14 + .../RetainedCutoverCoordinatorTest.java | 343 ++++++++++++++++++ .../RetainedCutoverFactoryRaceTest.java | 163 +++++++++ .../workflow/RetainedCutoverFailureTest.java | 288 +++++++++++++++ .../RetainedCutoverLifecycleTest.java | 199 ++++++++++ 15 files changed, 1758 insertions(+) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverErrorCode.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverException.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverOutcome.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverRelease.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverReleaseRequiredException.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverResult.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverState.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverSteps.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcFailedAcquireSettlement.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinatorTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverFactoryRaceTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverFailureTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverLifecycleTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java new file mode 100644 index 0000000000..0fd7adc442 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.time.Duration; +import java.util.Objects; +import java.util.function.LongSupplier; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceException; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceLease; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceOrchestrator; +import org.apache.hertzbeat.manager.setup.api.OperationIdValidator; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; + +/** + * Composes one provision-and-copy attempt while retaining successful maintenance ownership. + * + *

The work deadline does not bound exact target cleanup or maintenance release. Those safety + * operations may block, in which case this coordinator deliberately retains the single active + * operation and its fence. + */ +final class RetainedCutoverCoordinator { + + private final RetainedCutoverState state = new RetainedCutoverState(); + private final TargetJdbcConnectionFactory targetFactory; + private final RetainedCutoverSteps steps; + private final LongSupplier ticker; + + RetainedCutoverCoordinator( + TargetJdbcConnectionFactory targetFactory, + FlywayTargetSchemaProvisioner provisioner, + MigrationMaintenanceOrchestrator maintenance, + JdbcMetadataMigrationExecutor executor, + LongSupplier ticker) { + this.targetFactory = Objects.requireNonNull(targetFactory, "targetFactory"); + this.steps = new RetainedCutoverSteps( + targetFactory, provisioner, maintenance, executor); + this.ticker = Objects.requireNonNull(ticker, "ticker"); + } + + RetainedCutoverResult execute( + String operationId, + MetadataDatabaseSettings target, + SecretValue borrowedPassword, + Duration timeout, + MetadataMigrationProgressSink progress) { + requireRequest(operationId, target, borrowedPassword, timeout, progress); + JdbcMetadataMigrationDeadline deadline = JdbcMetadataMigrationDeadline.start(timeout, ticker); + RetainedCutoverState.Execution execution = state.reserve(operationId); + TargetJdbcConnectionLease provisionLease = acquire(execution, target, borrowedPassword, deadline); + String provisionIdentity = targetIdentity(execution, provisionLease, deadline); + execution.targetIdentityHash(provisionIdentity); + RetainedCutoverOutcome provisionOutcome = steps.provision( + provisionLease, target, deadline); + if (!provisionOutcome.successful()) { + return finish(execution, RetainedCutoverRelease.resources( + provisionLease, null, provisionOutcome, false), deadline); + } + closeBeforeContinuation(execution, provisionLease); + + TargetJdbcConnectionLease copyLease = acquire(execution, target, borrowedPassword, deadline); + String copyIdentity = targetIdentity(execution, copyLease, deadline); + if (!provisionIdentity.equals(copyIdentity)) { + return finish(execution, RetainedCutoverRelease.resources( + copyLease, null, RetainedCutoverOutcome.identityChanged(), false), deadline); + } + MigrationMaintenanceLease maintenanceLease; + try { + maintenanceLease = steps.acquireMaintenance(operationId, deadline); + } catch (RuntimeException | Error failure) { + return finish(execution, RetainedCutoverRelease.resources( + copyLease, null, RetainedCutoverOutcome.failure(failure), false), deadline); + } + RetainedCutoverOutcome copyOutcome = steps.copy( + copyLease, maintenanceLease, target, deadline, progress); + return finish(execution, RetainedCutoverRelease.resources( + copyLease, maintenanceLease, copyOutcome, copyOutcome.successful()), deadline); + } + + RetainedCutoverResult retained(String operationId) { + requireOperationId(operationId); + return state.retained(operationId); + } + + void releaseRetained(String operationId) { + requireOperationId(operationId); + RetainedCutoverState.Execution execution = state.claimRetainedRelease(operationId); + finish(execution, execution.release(), cleanupDeadline()); + } + + RetainedCutoverResult retryRelease(String operationId, Duration timeout) { + requireOperationId(operationId); + JdbcMetadataMigrationDeadline deadline = JdbcMetadataMigrationDeadline.start(timeout, ticker); + RetainedCutoverState.Execution execution = state.claimPendingRelease(operationId); + return finish(execution, execution.release(), deadline); + } + + private TargetJdbcConnectionLease acquire( + RetainedCutoverState.Execution execution, + MetadataDatabaseSettings target, + SecretValue password, + JdbcMetadataMigrationDeadline deadline) { + try { + return steps.acquire(target, password, deadline); + } catch (TargetJdbcConnectionException failure) { + if (mayHaveAsynchronousOwnership(failure.code())) { + RetainedCutoverOutcome outcome = RetainedCutoverOutcome.failure(failure); + releasePending(execution, RetainedCutoverRelease.factoryCleanup( + targetFactory, outcome)); + outcome.releaseRequired(); + } + state.clear(execution); + throw failure; + } catch (Error fatal) { + RetainedCutoverOutcome outcome = RetainedCutoverOutcome.failure(fatal); + releasePending(execution, RetainedCutoverRelease.factoryCleanup(targetFactory, outcome)); + outcome.releaseRequired(); + throw fatal; + } catch (RuntimeException unexpected) { + state.clear(execution); + throw new RetainedCutoverException(RetainedCutoverErrorCode.EXECUTION_FAILED); + } + } + + private String targetIdentity( + RetainedCutoverState.Execution execution, + TargetJdbcConnectionLease lease, + JdbcMetadataMigrationDeadline deadline) { + try { + return lease.targetIdentityHash(); + } catch (RuntimeException | Error failure) { + finish(execution, RetainedCutoverRelease.resources( + lease, null, RetainedCutoverOutcome.failure(failure), false), deadline); + throw new RetainedCutoverException(RetainedCutoverErrorCode.EXECUTION_FAILED); + } + } + + private void closeBeforeContinuation( + RetainedCutoverState.Execution execution, + TargetJdbcConnectionLease lease) { + boolean interrupted = Thread.interrupted(); + try { + lease.close(); + } catch (RuntimeException releaseFailure) { + RetainedCutoverOutcome outcome = RetainedCutoverOutcome.retryExecution(); + releasePending(execution, RetainedCutoverRelease.resources( + lease, null, outcome, false)); + outcome.releaseRequired(); + } catch (Error releaseFatal) { + RetainedCutoverRelease release = RetainedCutoverRelease.resources( + lease, null, RetainedCutoverOutcome.retryExecution(), false); + releasePending(execution, release); + RetainedCutoverOutcome.retryExecution().releaseFatal(releaseFatal); + } finally { + restoreInterrupt(interrupted | Thread.interrupted()); + } + } + + private RetainedCutoverResult finish( + RetainedCutoverState.Execution execution, + RetainedCutoverRelease release, + JdbcMetadataMigrationDeadline cleanupDeadline) { + boolean interrupted = Thread.interrupted(); + RetainedCutoverRelease.Advance advance; + try { + advance = release.advance(cleanupDeadline); + } catch (RuntimeException releaseFailure) { + releasePending(execution, release); + release.outcome().releaseRequired(); + throw releaseFailure; + } catch (Error releaseFatal) { + releasePending(execution, release); + release.outcome().releaseFatal(releaseFatal); + throw releaseFatal; + } finally { + restoreInterrupt(interrupted | Thread.interrupted()); + } + if (advance == RetainedCutoverRelease.Advance.RETAINED) { + return retain(execution, release.takeRetainedMaintenance()); + } + state.clear(execution); + release.outcome().replay(); + return execution.result(RetainedCutoverResult.Status.RELEASED); + } + + private RetainedCutoverResult retain( + RetainedCutoverState.Execution execution, MigrationMaintenanceLease maintenanceLease) { + return state.retain(execution, maintenanceLease); + } + + private void releasePending( + RetainedCutoverState.Execution execution, RetainedCutoverRelease release) { + state.releasePending(execution, release); + } + + private JdbcMetadataMigrationDeadline cleanupDeadline() { + return JdbcMetadataMigrationDeadline.start(Duration.ofSeconds(30), ticker); + } + + private static void requireRequest( + String operationId, + MetadataDatabaseSettings target, + SecretValue password, + Duration timeout, + MetadataMigrationProgressSink progress) { + requireOperationId(operationId); + Objects.requireNonNull(target, "target"); + Objects.requireNonNull(password, "password"); + Objects.requireNonNull(timeout, "timeout"); + Objects.requireNonNull(progress, "progress"); + } + + private static void requireOperationId(String operationId) { + if (!OperationIdValidator.isSafe(operationId)) { + throw MigrationMaintenanceException.invalidRequest(); + } + } + + private static void restoreInterrupt(boolean interrupted) { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + private static boolean mayHaveAsynchronousOwnership(TargetJdbcConnectionErrorCode code) { + return code == TargetJdbcConnectionErrorCode.TIMEOUT + || code == TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED + || code == TargetJdbcConnectionErrorCode.FACTORY_CLOSED + || code == TargetJdbcConnectionErrorCode.OPERATION_CONFLICT; + } + +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverErrorCode.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverErrorCode.java new file mode 100644 index 0000000000..f647acb91a --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverErrorCode.java @@ -0,0 +1,15 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +/** Stable secret-free failures owned by retained cutover composition. */ +enum RetainedCutoverErrorCode { + TARGET_IDENTITY_CHANGED, + PREPARATION_RETRY_REQUIRED, + EXECUTION_FAILED +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverException.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverException.java new file mode 100644 index 0000000000..95fbb7738a --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverException.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Locale; +import java.util.Objects; + +/** Cause-free retained cutover failure safe for a later workflow boundary. */ +final class RetainedCutoverException extends RuntimeException { + + private final RetainedCutoverErrorCode code; + + RetainedCutoverException(RetainedCutoverErrorCode code) { + super("Retained cutover failed: " + + Objects.requireNonNull(code, "code").name().toLowerCase(Locale.ROOT)); + this.code = code; + } + + RetainedCutoverErrorCode code() { + return code; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverOutcome.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverOutcome.java new file mode 100644 index 0000000000..d0d34f4b91 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverOutcome.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceException; + +/** Completed secret-free cutover result retained while exact cleanup is pending. */ +final class RetainedCutoverOutcome { + + private final RuntimeException stableFailure; + private final Error fatal; + + private RetainedCutoverOutcome(RuntimeException stableFailure, Error fatal) { + this.stableFailure = stableFailure; + this.fatal = fatal; + } + + static RetainedCutoverOutcome success() { + return new RetainedCutoverOutcome(null, null); + } + + static RetainedCutoverOutcome retryExecution() { + return stable(new RetainedCutoverException( + RetainedCutoverErrorCode.PREPARATION_RETRY_REQUIRED)); + } + + static RetainedCutoverOutcome identityChanged() { + return stable(new RetainedCutoverException( + RetainedCutoverErrorCode.TARGET_IDENTITY_CHANGED)); + } + + static RetainedCutoverOutcome factoryClosed() { + return stable(new TargetJdbcConnectionException( + TargetJdbcConnectionErrorCode.FACTORY_CLOSED)); + } + + static RetainedCutoverOutcome failure(Throwable failure) { + if (failure instanceof Error error) { + return new RetainedCutoverOutcome(null, error); + } + if (failure instanceof MetadataMigrationException + || failure instanceof TargetJdbcConnectionException + || failure instanceof TargetSchemaProvisioningException + || failure instanceof MigrationMaintenanceException + || failure instanceof RetainedCutoverException) { + return stable((RuntimeException) failure); + } + return stable(new RetainedCutoverException(RetainedCutoverErrorCode.EXECUTION_FAILED)); + } + + boolean successful() { + return stableFailure == null && fatal == null; + } + + RetainedCutoverOutcome terminalFactoryClosedUnlessFatal() { + return fatal == null ? factoryClosed() : this; + } + + void replay() { + if (fatal != null) { + throw fatal; + } + if (stableFailure != null) { + throw stableFailure; + } + } + + void releaseRequired() { + if (fatal != null) { + RetainedCutoverReleaseRequiredException.attach(fatal); + throw fatal; + } + throw new RetainedCutoverReleaseRequiredException(); + } + + void releaseFatal(Error releaseFatal) { + if (fatal != null) { + releaseRequired(); + } + RetainedCutoverReleaseRequiredException.attach(releaseFatal); + throw releaseFatal; + } + + private static RetainedCutoverOutcome stable(RuntimeException failure) { + return new RetainedCutoverOutcome(failure, null); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverRelease.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverRelease.java new file mode 100644 index 0000000000..4108108fdb --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverRelease.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceLease; + +/** Advances exact target cleanup before either retaining or releasing maintenance. */ +final class RetainedCutoverRelease { + + private RetainedCutoverOutcome outcome; + private final boolean retainMaintenanceOnSuccess; + private TargetJdbcConnectionFactory factoryCleanup; + private TargetJdbcConnectionLease targetLease; + private MigrationMaintenanceLease maintenanceLease; + + private RetainedCutoverRelease( + TargetJdbcConnectionFactory factoryCleanup, + TargetJdbcConnectionLease targetLease, + MigrationMaintenanceLease maintenanceLease, + RetainedCutoverOutcome outcome, + boolean retainMaintenanceOnSuccess) { + this.factoryCleanup = factoryCleanup; + this.targetLease = targetLease; + this.maintenanceLease = maintenanceLease; + this.outcome = outcome; + this.retainMaintenanceOnSuccess = retainMaintenanceOnSuccess; + } + + static RetainedCutoverRelease factoryCleanup( + TargetJdbcConnectionFactory factory, + RetainedCutoverOutcome outcome) { + return new RetainedCutoverRelease(factory, null, null, outcome, false); + } + + static RetainedCutoverRelease resources( + TargetJdbcConnectionLease targetLease, + MigrationMaintenanceLease maintenanceLease, + RetainedCutoverOutcome outcome, + boolean retainMaintenanceOnSuccess) { + return new RetainedCutoverRelease( + null, targetLease, maintenanceLease, outcome, retainMaintenanceOnSuccess); + } + + Advance advance(JdbcMetadataMigrationDeadline cleanupDeadline) { + if (factoryCleanup != null) { + TargetJdbcFailedAcquireSettlement settlement = + factoryCleanup.settleFailedAcquire(cleanupDeadline); + if (settlement == TargetJdbcFailedAcquireSettlement.TERMINAL_CLOSED) { + outcome = outcome.terminalFactoryClosedUnlessFatal(); + } + factoryCleanup = null; + } + if (targetLease != null) { + targetLease.close(); + targetLease = null; + } + if (retainMaintenanceOnSuccess && outcome.successful()) { + return Advance.RETAINED; + } + if (maintenanceLease != null) { + maintenanceLease.close(); + maintenanceLease = null; + } + return Advance.RELEASED; + } + + MigrationMaintenanceLease takeRetainedMaintenance() { + MigrationMaintenanceLease retained = maintenanceLease; + maintenanceLease = null; + return retained; + } + + RetainedCutoverOutcome outcome() { + return outcome; + } + + enum Advance { + RETAINED, + RELEASED + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverReleaseRequiredException.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverReleaseRequiredException.java new file mode 100644 index 0000000000..cdfcfb994e --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverReleaseRequiredException.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +/** Secret-free signal that exact cutover cleanup still belongs to the same operation. */ +final class RetainedCutoverReleaseRequiredException extends RuntimeException { + + RetainedCutoverReleaseRequiredException() { + super("Retained cutover release requires recovery"); + } + + static void attach(Error fatal) { + for (Throwable suppressed : fatal.getSuppressed()) { + if (suppressed instanceof RetainedCutoverReleaseRequiredException) { + return; + } + } + fatal.addSuppressed(new RetainedCutoverReleaseRequiredException()); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverResult.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverResult.java new file mode 100644 index 0000000000..c0d04c990b --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverResult.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.OperationIdValidator; + +/** Secret-free reference to a retained or explicitly released cutover capability. */ +record RetainedCutoverResult(String operationId, String targetIdentityHash, Status status) { + + RetainedCutoverResult { + if (!OperationIdValidator.isSafe(operationId)) { + throw new IllegalArgumentException("Invalid operation id"); + } + if (targetIdentityHash == null || !targetIdentityHash.matches("[0-9a-f]{64}")) { + throw new IllegalArgumentException("Invalid target identity"); + } + Objects.requireNonNull(status, "status"); + } + + enum Status { + RETAINED_SUCCESS, + ALREADY_RETAINED, + RELEASED + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverState.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverState.java new file mode 100644 index 0000000000..93646efdac --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverState.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceException; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceLease; + +/** Owns the single in-memory retained-cutover slot and its exact capability. */ +final class RetainedCutoverState { + + private Execution active; + + synchronized Execution reserve(String operationId) { + if (active != null) { + throw MigrationMaintenanceException.operationConflict(); + } + active = new Execution(operationId); + return active; + } + + synchronized RetainedCutoverResult retained(String operationId) { + Execution execution = require(operationId, Phase.RETAINED); + return execution.result(RetainedCutoverResult.Status.ALREADY_RETAINED); + } + + synchronized Execution claimRetainedRelease(String operationId) { + Execution execution = require(operationId, Phase.RETAINED); + execution.phase = Phase.RELEASING; + execution.release = RetainedCutoverRelease.resources( + null, execution.maintenanceLease, RetainedCutoverOutcome.success(), false); + execution.maintenanceLease = null; + return execution; + } + + synchronized Execution claimPendingRelease(String operationId) { + Execution execution = require(operationId, Phase.RELEASE_PENDING); + execution.phase = Phase.RELEASING; + return execution; + } + + synchronized void releasePending(Execution execution, RetainedCutoverRelease release) { + if (active == execution) { + execution.release = release; + execution.phase = Phase.RELEASE_PENDING; + } + } + + synchronized RetainedCutoverResult retain( + Execution execution, MigrationMaintenanceLease maintenanceLease) { + if (active != execution) { + throw MigrationMaintenanceException.operationConflict(); + } + execution.maintenanceLease = Objects.requireNonNull(maintenanceLease, "maintenanceLease"); + execution.release = null; + execution.phase = Phase.RETAINED; + return execution.result(RetainedCutoverResult.Status.RETAINED_SUCCESS); + } + + synchronized void clear(Execution execution) { + if (active == execution) { + active = null; + } + } + + private Execution require(String operationId, Phase phase) { + if (active == null + || active.phase != phase + || !active.operationId.equals(operationId)) { + throw MigrationMaintenanceException.operationConflict(); + } + return active; + } + + static final class Execution { + + private final String operationId; + private String targetIdentityHash; + private MigrationMaintenanceLease maintenanceLease; + private RetainedCutoverRelease release; + private Phase phase = Phase.EXECUTING; + + private Execution(String operationId) { + this.operationId = operationId; + } + + void targetIdentityHash(String targetIdentityHash) { + this.targetIdentityHash = targetIdentityHash; + } + + RetainedCutoverRelease release() { + return release; + } + + RetainedCutoverResult result(RetainedCutoverResult.Status status) { + return new RetainedCutoverResult(operationId, targetIdentityHash, status); + } + } + + private enum Phase { + EXECUTING, + RETAINED, + RELEASING, + RELEASE_PENDING + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverSteps.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverSteps.java new file mode 100644 index 0000000000..b8a8ec0cee --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverSteps.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceException; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceLease; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceOrchestrator; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; + +/** Executes scoped provision and copy steps without owning workflow state or cleanup. */ +final class RetainedCutoverSteps { + + private final TargetJdbcConnectionFactory targetFactory; + private final FlywayTargetSchemaProvisioner provisioner; + private final MigrationMaintenanceOrchestrator maintenance; + private final JdbcMetadataMigrationExecutor executor; + + RetainedCutoverSteps( + TargetJdbcConnectionFactory targetFactory, + FlywayTargetSchemaProvisioner provisioner, + MigrationMaintenanceOrchestrator maintenance, + JdbcMetadataMigrationExecutor executor) { + this.targetFactory = Objects.requireNonNull(targetFactory, "targetFactory"); + this.provisioner = Objects.requireNonNull(provisioner, "provisioner"); + this.maintenance = Objects.requireNonNull(maintenance, "maintenance"); + this.executor = Objects.requireNonNull(executor, "executor"); + } + + TargetJdbcConnectionLease acquire( + MetadataDatabaseSettings target, + SecretValue password, + JdbcMetadataMigrationDeadline deadline) { + return targetFactory.acquire(target, password, deadline); + } + + RetainedCutoverOutcome provision( + TargetJdbcConnectionLease lease, + MetadataDatabaseSettings target, + JdbcMetadataMigrationDeadline deadline) { + try { + lease.withConnection(connection -> provisioner.provision( + connection, target.kind(), deadline)); + return RetainedCutoverOutcome.success(); + } catch (RuntimeException | Error failure) { + return RetainedCutoverOutcome.failure(failure); + } + } + + MigrationMaintenanceLease acquireMaintenance( + String operationId, JdbcMetadataMigrationDeadline deadline) { + MigrationMaintenanceLease lease = maintenance.acquire( + operationId, deadline.remainingDuration()); + if (lease == null) { + throw MigrationMaintenanceException.maintenanceFailure(); + } + return lease; + } + + RetainedCutoverOutcome copy( + TargetJdbcConnectionLease targetLease, + MigrationMaintenanceLease maintenanceLease, + MetadataDatabaseSettings target, + JdbcMetadataMigrationDeadline deadline, + MetadataMigrationProgressSink progress) { + try { + targetLease.withConnection(targetConnection -> + maintenanceLease.withSourceConnection(sourceConnection -> + executor.execute(sourceConnection, targetConnection, + target.kind(), deadline, progress))); + return RetainedCutoverOutcome.success(); + } catch (RuntimeException | Error failure) { + return RetainedCutoverOutcome.failure(failure); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionFactory.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionFactory.java index 8b09c94cc0..0d5c243fa2 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionFactory.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcConnectionFactory.java @@ -105,6 +105,28 @@ final class TargetJdbcConnectionFactory implements AutoCloseable, TargetJdbcConn cleanupLane.retry(deadline); } + TargetJdbcFailedAcquireSettlement settleFailedAcquire( + JdbcMetadataMigrationDeadline deadline) { + awaitInactive(deadline); + TargetJdbcConnectionErrorCode cleanupState = cleanupLane.acquisitionFailure(); + if (cleanupState == TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED) { + cleanupLane.retry(deadline); + } + synchronized (this) { + if (lateFailure != null) { + throw lateFailure; + } + cleanupState = cleanupLane.acquisitionFailure(); + if (closed || cleanupState == TargetJdbcConnectionErrorCode.FACTORY_CLOSED) { + return TargetJdbcFailedAcquireSettlement.TERMINAL_CLOSED; + } + if (cleanupState != null) { + throw failure(cleanupState); + } + return TargetJdbcFailedAcquireSettlement.REUSABLE; + } + } + @Override public synchronized void close() { closed = true; @@ -138,12 +160,35 @@ final class TargetJdbcConnectionFactory implements AutoCloseable, TargetJdbcConn public synchronized void finished(TargetJdbcConnectionAttempt attempt) { if (active == attempt) { active = null; + notifyAll(); if (closed) { cleanupLane.close(); } } } + private synchronized void awaitInactive(JdbcMetadataMigrationDeadline deadline) { + boolean interrupted = false; + try { + while (active != null) { + long remaining = deadline.remainingNanos(); + if (remaining <= 0) { + throw failure(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED); + } + try { + TimeUnit.NANOSECONDS.timedWait(this, remaining); + } catch (InterruptedException ignored) { + interrupted = true; + throw failure(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED); + } + } + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + private synchronized void claim(TargetJdbcConnectionAttempt attempt) { if (lateFailure != null) { throw lateFailure; diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcFailedAcquireSettlement.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcFailedAcquireSettlement.java new file mode 100644 index 0000000000..ec58f3b57e --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcFailedAcquireSettlement.java @@ -0,0 +1,14 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +/** Stable state after a failed target acquisition has quiesced and exact cleanup has settled. */ +enum TargetJdbcFailedAcquireSettlement { + REUSABLE, + TERMINAL_CLOSED +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinatorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinatorTest.java new file mode 100644 index 0000000000..76f6d26396 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinatorTest.java @@ -0,0 +1,343 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.same; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceErrorCode; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceException; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceLease; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceOrchestrator; +import org.apache.hertzbeat.manager.maintenance.MigrationSourceAction; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.mockito.InOrder; +import org.mockito.ArgumentCaptor; + +@Timeout(15) +class RetainedCutoverCoordinatorTest { + + private static final String OPERATION_ID = "operation-a"; + private static final String IDENTITY = "a".repeat(64); + private static final String OTHER_IDENTITY = "b".repeat(64); + private static final Duration TIMEOUT = Duration.ofNanos(100); + private static final MetadataDatabaseSettings TARGET = new MetadataDatabaseSettings( + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db.example/hertzbeat", "migration"); + + @Test + void provisionsAndClosesOneLeaseBeforeFreshIdentityMatchedCopyAndRetainsMaintenance() { + Fixture fixture = new Fixture(); + + RetainedCutoverResult result = fixture.execute(); + + assertThat(result.status()).isEqualTo(RetainedCutoverResult.Status.RETAINED_SUCCESS); + assertThat(result.operationId()).isEqualTo(OPERATION_ID); + assertThat(result.targetIdentityHash()).isEqualTo(IDENTITY); + InOrder order = inOrder( + fixture.factory, fixture.provisionLease, fixture.provisioner, + fixture.copyLease, fixture.maintenance, fixture.maintenanceLease, fixture.executor); + order.verify(fixture.factory).acquire(same(TARGET), same(fixture.password), anyDeadline()); + order.verify(fixture.provisionLease).withConnection(any()); + order.verify(fixture.provisioner).provision( + same(fixture.provisionConnection), eq(MetadataDatabaseKind.POSTGRESQL), anyDeadline()); + order.verify(fixture.provisionLease).close(); + order.verify(fixture.factory).acquire(same(TARGET), same(fixture.password), anyDeadline()); + order.verify(fixture.maintenance).acquire(eq(OPERATION_ID), any()); + order.verify(fixture.copyLease).withConnection(any()); + order.verify(fixture.maintenanceLease).withSourceConnection(any()); + order.verify(fixture.executor).execute( + same(fixture.sourceConnection), same(fixture.copyConnection), + eq(MetadataDatabaseKind.POSTGRESQL), anyDeadline(), + same(MetadataMigrationProgressSink.NO_OP)); + order.verify(fixture.copyLease).close(); + verify(fixture.maintenanceLease, never()).close(); + verify(fixture.factory, never()).close(); + assertThat(fixture.provisionConnection).isNotSameAs(fixture.copyConnection); + } + + @Test + void passesOneRootDeadlineAndConsumesMaintenanceAcquisitionFromItsRemainingBudget() { + Fixture fixture = new Fixture(); + fixture.ticker.set(10); + when(fixture.maintenance.acquire(eq(OPERATION_ID), any())).thenAnswer(invocation -> { + assertThat((Duration) invocation.getArgument(1)).isEqualTo(Duration.ofNanos(55)); + fixture.ticker.addAndGet(20); + return fixture.maintenanceLease; + }); + doAnswer(invocation -> { + JdbcMetadataMigrationDeadline deadline = invocation.getArgument(2); + assertThat(deadline.remainingNanos()).isEqualTo(90); + fixture.ticker.addAndGet(20); + return new TargetSchemaProvisioningOutcome(TargetSchemaConnectionDisposition.REUSABLE); + }).when(fixture.provisioner).provision(any(), any(), anyDeadline()); + when(fixture.factory.acquire(same(TARGET), same(fixture.password), anyDeadline())) + .thenAnswer(invocation -> { + JdbcMetadataMigrationDeadline deadline = invocation.getArgument(2); + assertThat(deadline.remainingNanos()).isPositive(); + fixture.ticker.addAndGet(10); + return fixture.provisionLease; + }) + .thenAnswer(invocation -> { + JdbcMetadataMigrationDeadline deadline = invocation.getArgument(2); + assertThat(deadline.remainingNanos()).isEqualTo(70); + fixture.ticker.addAndGet(15); + return fixture.copyLease; + }); + doAnswer(invocation -> { + JdbcMetadataMigrationDeadline deadline = invocation.getArgument(3); + assertThat(deadline.remainingNanos()).isEqualTo(35); + return null; + }).when(fixture.executor).execute(any(), any(), any(), anyDeadline(), any()); + + fixture.execute(); + + ArgumentCaptor acquireDeadlines = + ArgumentCaptor.forClass(JdbcMetadataMigrationDeadline.class); + verify(fixture.factory, times(2)).acquire( + same(TARGET), same(fixture.password), acquireDeadlines.capture()); + ArgumentCaptor provisionDeadline = + ArgumentCaptor.forClass(JdbcMetadataMigrationDeadline.class); + verify(fixture.provisioner).provision( + any(), any(), provisionDeadline.capture()); + ArgumentCaptor copyDeadline = + ArgumentCaptor.forClass(JdbcMetadataMigrationDeadline.class); + verify(fixture.executor).execute( + any(), any(), any(), copyDeadline.capture(), any()); + assertThat(acquireDeadlines.getAllValues()) + .allSatisfy(value -> assertThat(value).isSameAs(provisionDeadline.getValue())); + assertThat(copyDeadline.getValue()).isSameAs(provisionDeadline.getValue()); + verify(fixture.executor).execute(any(), any(), any(), anyDeadline(), any()); + } + + @Test + void retainedStatusReplaysWithoutCredentialsOrWorkAndExecuteRemainsExclusive() { + Fixture fixture = new Fixture(); + RetainedCutoverResult first = fixture.execute(); + + RetainedCutoverResult replay = fixture.coordinator.retained(OPERATION_ID); + + assertThat(replay.status()).isEqualTo(RetainedCutoverResult.Status.ALREADY_RETAINED); + assertThat(replay.targetIdentityHash()).isEqualTo(first.targetIdentityHash()); + verify(fixture.factory, times(2)).acquire(any(), any(), anyDeadline()); + verify(fixture.executor).execute(any(), any(), any(), anyDeadline(), any()); + assertConflict(fixture::execute); + assertConflict(() -> fixture.coordinator.execute( + "operation-b", TARGET, fixture.password, TIMEOUT, MetadataMigrationProgressSink.NO_OP)); + } + + @Test + void identityChangeClosesFreshTargetWithoutAcquiringMaintenanceOrCopying() { + Fixture fixture = new Fixture(IDENTITY, OTHER_IDENTITY); + + assertThatThrownBy(fixture::execute) + .isInstanceOfSatisfying(RetainedCutoverException.class, failure -> + assertThat(failure.code()) + .isEqualTo(RetainedCutoverErrorCode.TARGET_IDENTITY_CHANGED)); + + verify(fixture.copyLease).close(); + verifyNoInteractions(fixture.maintenance, fixture.maintenanceLease, fixture.executor); + } + + @Test + void copyFailureClosesTargetThenMaintenanceBeforeReplayingStableFailure() { + Fixture fixture = new Fixture(); + doThrow(new MetadataMigrationException(MetadataMigrationErrorCode.VERIFICATION)) + .when(fixture.executor).execute(any(), any(), any(), anyDeadline(), any()); + + assertThatThrownBy(fixture::execute) + .isInstanceOfSatisfying(MetadataMigrationException.class, failure -> + assertThat(failure.code()).isEqualTo(MetadataMigrationErrorCode.VERIFICATION)); + + InOrder release = inOrder(fixture.copyLease, fixture.maintenanceLease); + release.verify(fixture.copyLease).close(); + release.verify(fixture.maintenanceLease).close(); + } + + @Test + void failedTargetCloseRetainsExactFailureAndSameOperationRetryNeverRecopies() { + Fixture fixture = new Fixture(); + doThrow(new MetadataMigrationException(MetadataMigrationErrorCode.COPY)) + .when(fixture.executor).execute(any(), any(), any(), anyDeadline(), any()); + doThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED)) + .doNothing().when(fixture.copyLease).close(); + + assertThatThrownBy(fixture::execute) + .isInstanceOf(RetainedCutoverReleaseRequiredException.class) + .hasNoCause(); + assertConflict(() -> fixture.coordinator.retryRelease("operation-b", Duration.ofSeconds(1))); + + assertThatThrownBy(() -> fixture.coordinator.retryRelease(OPERATION_ID, Duration.ofSeconds(1))) + .isInstanceOfSatisfying(MetadataMigrationException.class, failure -> + assertThat(failure.code()).isEqualTo(MetadataMigrationErrorCode.COPY)); + verify(fixture.executor).execute(any(), any(), any(), anyDeadline(), any()); + verify(fixture.copyLease, times(2)).close(); + verify(fixture.maintenanceLease).close(); + } + + @Test + void successfulCopyWhoseTargetCloseNeedsRetryConvergesToRetainedSuccess() { + Fixture fixture = new Fixture(); + doThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED)) + .doNothing().when(fixture.copyLease).close(); + + assertThatThrownBy(fixture::execute) + .isInstanceOf(RetainedCutoverReleaseRequiredException.class); + + RetainedCutoverResult result = fixture.coordinator.retryRelease( + OPERATION_ID, Duration.ofSeconds(1)); + assertThat(result.status()).isEqualTo(RetainedCutoverResult.Status.RETAINED_SUCCESS); + verify(fixture.executor).execute(any(), any(), any(), anyDeadline(), any()); + verify(fixture.maintenanceLease, never()).close(); + } + + @Test + void maintenanceReleaseFailureRetainsExactLeaseAndRetryDoesNotRepeatCopy() { + Fixture fixture = new Fixture(); + doThrow(new MetadataMigrationException(MetadataMigrationErrorCode.COPY)) + .when(fixture.executor).execute(any(), any(), any(), anyDeadline(), any()); + doThrow(MigrationMaintenanceException.maintenanceFailure()) + .doNothing().when(fixture.maintenanceLease).close(); + + assertThatThrownBy(fixture::execute) + .isInstanceOf(RetainedCutoverReleaseRequiredException.class); + + assertThatThrownBy(() -> fixture.coordinator.retryRelease(OPERATION_ID, Duration.ofSeconds(1))) + .isInstanceOfSatisfying(MetadataMigrationException.class, failure -> + assertThat(failure.code()).isEqualTo(MetadataMigrationErrorCode.COPY)); + verify(fixture.executor).execute(any(), any(), any(), anyDeadline(), any()); + verify(fixture.maintenanceLease, times(2)).close(); + } + + @Test + void fatalCopyRemainsPrimaryWhenCleanupNeedsSameOperationRetry() { + Fixture fixture = new Fixture(); + AssertionError fatal = new AssertionError("fatal copy"); + doThrow(fatal).when(fixture.executor).execute(any(), any(), any(), anyDeadline(), any()); + doThrow(new IllegalStateException("private close")) + .doNothing().when(fixture.copyLease).close(); + + assertThatThrownBy(fixture::execute).isSameAs(fatal); + assertThat(fatal.getSuppressed()).singleElement() + .isInstanceOf(RetainedCutoverReleaseRequiredException.class); + assertThatThrownBy(() -> fixture.coordinator.retryRelease(OPERATION_ID, Duration.ofSeconds(1))) + .isSameAs(fatal); + verify(fixture.executor).execute(any(), any(), any(), anyDeadline(), any()); + } + + @Test + void mandatoryFailureCleanupClearsAndRestoresInterrupt() { + Fixture fixture = new Fixture(); + doAnswer(invocation -> { + Thread.currentThread().interrupt(); + throw new MetadataMigrationException(MetadataMigrationErrorCode.COPY); + }).when(fixture.executor).execute(any(), any(), any(), anyDeadline(), any()); + doAnswer(invocation -> { + assertThat(Thread.currentThread().isInterrupted()).isFalse(); + return null; + }).when(fixture.copyLease).close(); + doAnswer(invocation -> { + assertThat(Thread.currentThread().isInterrupted()).isFalse(); + return null; + }).when(fixture.maintenanceLease).close(); + + try { + assertThatThrownBy(fixture::execute).isInstanceOf(MetadataMigrationException.class); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } finally { + Thread.interrupted(); + } + } + + private static JdbcMetadataMigrationDeadline anyDeadline() { + return any(JdbcMetadataMigrationDeadline.class); + } + + private static void assertConflict(Runnable action) { + assertThatThrownBy(action::run) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, failure -> + assertThat(failure.code()) + .isEqualTo(MigrationMaintenanceErrorCode.MIGRATION_OPERATION_CONFLICT)); + } + + private static final class Fixture { + + private final Connection provisionConnection = mock(Connection.class); + private final Connection copyConnection = mock(Connection.class); + private final Connection sourceConnection = mock(Connection.class); + private final TargetJdbcConnectionFactory factory = mock(TargetJdbcConnectionFactory.class); + private final TargetJdbcConnectionLease provisionLease = mock(TargetJdbcConnectionLease.class); + private final TargetJdbcConnectionLease copyLease = mock(TargetJdbcConnectionLease.class); + private final FlywayTargetSchemaProvisioner provisioner = mock(FlywayTargetSchemaProvisioner.class); + private final MigrationMaintenanceOrchestrator maintenance = mock(MigrationMaintenanceOrchestrator.class); + private final MigrationMaintenanceLease maintenanceLease = mock(MigrationMaintenanceLease.class); + private final JdbcMetadataMigrationExecutor executor = mock(JdbcMetadataMigrationExecutor.class); + private final SecretValue password = mock(SecretValue.class); + private final AtomicLong ticker = new AtomicLong(); + private final RetainedCutoverCoordinator coordinator; + + private Fixture() { + this(IDENTITY, IDENTITY); + } + + private Fixture(String provisionIdentity, String copyIdentity) { + when(provisionLease.targetIdentityHash()).thenReturn(provisionIdentity); + when(copyLease.targetIdentityHash()).thenReturn(copyIdentity); + when(factory.acquire(same(TARGET), same(password), anyDeadline())) + .thenReturn(provisionLease, copyLease); + scopedTarget(provisionLease, provisionConnection); + scopedTarget(copyLease, copyConnection); + scopedSource(maintenanceLease, sourceConnection); + when(provisioner.provision(any(), any(), anyDeadline())) + .thenReturn(new TargetSchemaProvisioningOutcome( + TargetSchemaConnectionDisposition.REUSABLE)); + when(maintenance.acquire(eq(OPERATION_ID), any())).thenReturn(maintenanceLease); + coordinator = new RetainedCutoverCoordinator( + factory, provisioner, maintenance, executor, ticker::get); + } + + private RetainedCutoverResult execute() { + return coordinator.execute( + OPERATION_ID, TARGET, password, TIMEOUT, MetadataMigrationProgressSink.NO_OP); + } + + private static void scopedTarget(TargetJdbcConnectionLease lease, Connection connection) { + doAnswer(invocation -> { + TargetJdbcConnectionAction action = invocation.getArgument(0); + action.execute(connection); + return null; + }).when(lease).withConnection(any()); + } + + private static void scopedSource(MigrationMaintenanceLease lease, Connection connection) { + doAnswer(invocation -> { + MigrationSourceAction action = invocation.getArgument(0); + action.execute(connection); + return null; + }).when(lease).withSourceConnection(any()); + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverFactoryRaceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverFactoryRaceTest.java new file mode 100644 index 0000000000..1f54b78299 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverFactoryRaceTest.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceOrchestrator; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +@Timeout(15) +class RetainedCutoverFactoryRaceTest { + + private static final String OPERATION_A = "operation-a"; + private static final String OPERATION_B = "operation-b"; + private static final Duration TIMEOUT = Duration.ofSeconds(5); + private static final MetadataDatabaseSettings TARGET = new MetadataDatabaseSettings( + MetadataDatabaseKind.MYSQL, "jdbc:mysql://db.example/hertzbeat", "migration"); + + @Test + void timedOutAcquireKeepsLateCleanupBoundToTheOriginalOperation() throws Exception { + CountDownLatch connectorEntered = new CountDownLatch(1); + CountDownLatch releaseConnector = new CountDownLatch(1); + CountDownLatch firstCloseAttempted = new CountDownLatch(1); + AtomicInteger closes = new AtomicInteger(); + Connection connection = mysqlConnection(); + doAnswer(invocation -> { + firstCloseAttempted.countDown(); + if (closes.getAndIncrement() == 0) { + throw new SQLException("private close diagnostic"); + } + return null; + }).when(connection).close(); + TargetJdbcConnector connector = (target, username, password, deadline) -> { + connectorEntered.countDown(); + awaitIgnoringInterrupt(releaseConnector); + return connection; + }; + TargetJdbcResultWaiter timeoutAfterConnectorStarts = (ready, remaining) -> { + assertThat(connectorEntered.await(1, TimeUnit.SECONDS)).isTrue(); + return false; + }; + FlywayTargetSchemaProvisioner provisioner = mock(FlywayTargetSchemaProvisioner.class); + MigrationMaintenanceOrchestrator maintenance = mock(MigrationMaintenanceOrchestrator.class); + JdbcMetadataMigrationExecutor executor = mock(JdbcMetadataMigrationExecutor.class); + try (TargetJdbcConnectionFactory factory = new TargetJdbcConnectionFactory( + worker(), worker(), Runnable::run, connector, + new TargetJdbcConnectionVerifier(Runnable::run), timeoutAfterConnectorStarts); + SecretValue password = SecretValue.of("borrowed-password")) { + RetainedCutoverCoordinator coordinator = new RetainedCutoverCoordinator( + factory, provisioner, maintenance, executor, System::nanoTime); + + assertThatThrownBy(() -> execute(coordinator, OPERATION_A, password)) + .isInstanceOf(RetainedCutoverReleaseRequiredException.class); + assertConflict(() -> execute(coordinator, OPERATION_B, password)); + + releaseConnector.countDown(); + assertThat(firstCloseAttempted.await(2, TimeUnit.SECONDS)).isTrue(); + assertConflict(() -> execute(coordinator, OPERATION_B, password)); + assertThatThrownBy(() -> coordinator.retryRelease(OPERATION_A, TIMEOUT)) + .isInstanceOfSatisfying(TargetJdbcConnectionException.class, failure -> + assertThat(failure.code()) + .isEqualTo(TargetJdbcConnectionErrorCode.FACTORY_CLOSED)); + + verify(connection, times(2)).close(); + verifyNoInteractions(provisioner, maintenance, executor); + } finally { + releaseConnector.countDown(); + } + } + + @Test + void terminalFactorySettlementNeverOverwritesTheOriginalAcquisitionFatal() { + AssertionError fatal = new AssertionError("private connector fatal"); + TargetJdbcConnector connector = (target, username, password, deadline) -> { + throw fatal; + }; + FlywayTargetSchemaProvisioner provisioner = mock(FlywayTargetSchemaProvisioner.class); + MigrationMaintenanceOrchestrator maintenance = mock(MigrationMaintenanceOrchestrator.class); + JdbcMetadataMigrationExecutor executor = mock(JdbcMetadataMigrationExecutor.class); + try (TargetJdbcConnectionFactory factory = new TargetJdbcConnectionFactory(connector, Runnable::run); + SecretValue password = SecretValue.of("borrowed-password")) { + RetainedCutoverCoordinator coordinator = new RetainedCutoverCoordinator( + factory, provisioner, maintenance, executor, System::nanoTime); + + assertThatThrownBy(() -> execute(coordinator, OPERATION_A, password)).isSameAs(fatal); + assertThat(fatal.getSuppressed()).singleElement() + .isInstanceOf(RetainedCutoverReleaseRequiredException.class); + assertThatThrownBy(() -> coordinator.retryRelease(OPERATION_A, TIMEOUT)).isSameAs(fatal); + + verifyNoInteractions(provisioner, maintenance, executor); + } + } + + private static RetainedCutoverResult execute( + RetainedCutoverCoordinator coordinator, String operationId, SecretValue password) { + return coordinator.execute( + operationId, TARGET, password, TIMEOUT, MetadataMigrationProgressSink.NO_OP); + } + + private static void assertConflict(Runnable action) { + assertThatThrownBy(action::run) + .isInstanceOf(org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceException.class); + } + + private static Connection mysqlConnection() throws SQLException { + Connection connection = mock(Connection.class); + DatabaseMetaData metadata = mock(DatabaseMetaData.class); + when(connection.getAutoCommit()).thenReturn(true); + when(connection.isReadOnly()).thenReturn(false); + when(connection.getMetaData()).thenReturn(metadata); + when(metadata.getDatabaseProductName()).thenReturn("MySQL"); + when(metadata.getURL()).thenReturn("jdbc:mysql://db.example/hertzbeat"); + when(connection.getCatalog()).thenReturn("hertzbeat"); + return connection; + } + + private static ThreadPoolExecutor worker() { + ThreadPoolExecutor worker = new ThreadPoolExecutor( + 0, 1, 30, TimeUnit.SECONDS, new SynchronousQueue<>()); + worker.allowCoreThreadTimeOut(true); + return worker; + } + + private static void awaitIgnoringInterrupt(CountDownLatch latch) { + boolean interrupted = false; + while (true) { + try { + latch.await(); + break; + } catch (InterruptedException ignored) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverFailureTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverFailureTest.java new file mode 100644 index 0000000000..175c7582e6 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverFailureTest.java @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.same; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.time.Duration; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceException; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceLease; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceOrchestrator; +import org.apache.hertzbeat.manager.maintenance.MigrationSourceAction; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +@Timeout(15) +class RetainedCutoverFailureTest { + + private static final String OPERATION_ID = "operation-a"; + private static final String IDENTITY = "a".repeat(64); + private static final Duration TIMEOUT = Duration.ofSeconds(1); + private static final MetadataDatabaseSettings TARGET = new MetadataDatabaseSettings( + MetadataDatabaseKind.MYSQL, "jdbc:mysql://db.example/hertzbeat", "migration"); + + @Test + void stableProvisionFailureClosesOnlyProvisionLeaseAndReplaysTheSameFailure() { + Fixture fixture = new Fixture(); + TargetSchemaProvisioningException failure = provisioningFailure(); + doThrow(failure).when(fixture.provisioner).provision(any(), any(), anyDeadline()); + + assertThatThrownBy(fixture::execute).isSameAs(failure); + + verify(fixture.provisionLease).close(); + verify(fixture.factory).acquire(any(), any(), anyDeadline()); + verifyNoInteractions(fixture.maintenance, fixture.executor); + } + + @Test + void fatalProvisionAndFailedCloseRetainFatalAndRetryNeverReprovisions() { + Fixture fixture = new Fixture(); + AssertionError fatal = new AssertionError("fatal provision"); + doThrow(fatal).when(fixture.provisioner).provision(any(), any(), anyDeadline()); + doThrow(new IllegalStateException("private close")) + .doNothing().when(fixture.provisionLease).close(); + + assertThatThrownBy(fixture::execute).isSameAs(fatal); + assertThat(fatal.getSuppressed()).singleElement() + .isInstanceOf(RetainedCutoverReleaseRequiredException.class); + + assertThatThrownBy(() -> fixture.coordinator.retryRelease(OPERATION_ID, TIMEOUT)) + .isSameAs(fatal); + verify(fixture.provisioner).provision(any(), any(), anyDeadline()); + verify(fixture.factory).acquire(any(), any(), anyDeadline()); + verify(fixture.provisionLease, times(2)).close(); + } + + @Test + void interruptedProvisionFailureClearsInterruptForCloseThenRestoresIt() { + Fixture fixture = new Fixture(); + TargetSchemaProvisioningException failure = provisioningFailure(); + doAnswer(invocation -> { + Thread.currentThread().interrupt(); + throw failure; + }).when(fixture.provisioner).provision(any(), any(), anyDeadline()); + doAnswer(invocation -> { + assertThat(Thread.currentThread().isInterrupted()).isFalse(); + return null; + }).when(fixture.provisionLease).close(); + + try { + assertThatThrownBy(fixture::execute).isSameAs(failure); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } finally { + Thread.interrupted(); + } + } + + @Test + void secondAcquireFailureOccursAfterProvisionCloseAndNeverAcquiresMaintenance() { + Fixture fixture = new Fixture(); + TargetJdbcConnectionException unavailable = + new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.UNAVAILABLE); + when(fixture.factory.acquire(same(TARGET), same(fixture.password), anyDeadline())) + .thenReturn(fixture.provisionLease) + .thenThrow(unavailable); + + assertThatThrownBy(fixture::execute).isSameAs(unavailable); + + verify(fixture.provisionLease).close(); + verifyNoInteractions(fixture.maintenance, fixture.executor); + } + + @Test + void maintenanceAcquireFailureClosesCopyTargetBeforeReplayingFailure() { + Fixture fixture = new Fixture(); + MigrationMaintenanceException failure = MigrationMaintenanceException.sourceUnavailable(); + when(fixture.maintenance.acquire(eq(OPERATION_ID), any())).thenThrow(failure); + + assertThatThrownBy(fixture::execute).isSameAs(failure); + + verify(fixture.copyLease).close(); + verifyNoInteractions(fixture.executor); + } + + @Test + void provisionalFactoryCleanupIsBoundToOperationAndSettlesAsTerminal() { + Fixture fixture = new Fixture(); + when(fixture.factory.acquire(any(), any(), anyDeadline())) + .thenThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED)); + when(fixture.factory.settleFailedAcquire(anyDeadline())) + .thenReturn(TargetJdbcFailedAcquireSettlement.TERMINAL_CLOSED); + + assertThatThrownBy(fixture::execute) + .isInstanceOf(RetainedCutoverReleaseRequiredException.class); + assertConflict(() -> fixture.coordinator.retryRelease("operation-b", TIMEOUT)); + + assertThatThrownBy(() -> fixture.coordinator.retryRelease(OPERATION_ID, TIMEOUT)) + .isInstanceOfSatisfying(TargetJdbcConnectionException.class, failure -> + assertThat(failure.code()) + .isEqualTo(TargetJdbcConnectionErrorCode.FACTORY_CLOSED)); + verify(fixture.factory).acquire(any(), any(), anyDeadline()); + verify(fixture.factory).settleFailedAcquire(anyDeadline()); + verifyNoInteractions(fixture.provisioner, fixture.maintenance, fixture.executor); + } + + @Test + void retainedSuccessCanBeExplicitlyReleasedAndReleaseFailureUsesExactSameOpRetry() { + Fixture fixture = new Fixture(); + fixture.execute(); + doThrow(MigrationMaintenanceException.maintenanceFailure()) + .doNothing().when(fixture.maintenanceLease).close(); + + assertThatThrownBy(() -> fixture.coordinator.releaseRetained(OPERATION_ID)) + .isInstanceOf(RetainedCutoverReleaseRequiredException.class); + assertConflict(() -> fixture.coordinator.releaseRetained("operation-b")); + + fixture.coordinator.retryRelease(OPERATION_ID, TIMEOUT); + verify(fixture.executor).execute(any(), any(), any(), anyDeadline(), any()); + verify(fixture.maintenanceLease, times(2)).close(); + assertConflict(() -> fixture.coordinator.retained(OPERATION_ID)); + } + + @Test + void invalidDeadlineDoesNotRetainTheOperationSlot() { + Fixture fixture = new Fixture(); + + assertThatThrownBy(() -> fixture.coordinator.execute( + OPERATION_ID, + TARGET, + fixture.password, + Duration.ZERO, + MetadataMigrationProgressSink.NO_OP)) + .isInstanceOf(MetadataMigrationException.class); + + assertThat(fixture.execute().status()).isEqualTo(RetainedCutoverResult.Status.RETAINED_SUCCESS); + } + + @Test + void provisionIdentityFatalClosesExactLeaseBeforeReplayingFatal() { + Fixture fixture = new Fixture(); + AssertionError fatal = new AssertionError("identity fatal"); + when(fixture.provisionLease.targetIdentityHash()).thenThrow(fatal); + + assertThatThrownBy(fixture::execute).isSameAs(fatal); + + verify(fixture.provisionLease).close(); + verifyNoInteractions(fixture.provisioner, fixture.maintenance, fixture.executor); + } + + @Test + void provisionIdentityRuntimeClosesExactLeaseAndUsesStableFailure() { + Fixture fixture = new Fixture(); + when(fixture.provisionLease.targetIdentityHash()).thenThrow(new IllegalStateException("private")); + + assertThatThrownBy(fixture::execute) + .isInstanceOfSatisfying(RetainedCutoverException.class, failure -> + assertThat(failure.code()).isEqualTo(RetainedCutoverErrorCode.EXECUTION_FAILED)) + .hasNoCause(); + + verify(fixture.provisionLease).close(); + verifyNoInteractions(fixture.provisioner, fixture.maintenance, fixture.executor); + } + + @Test + void factoryFatalRemainsBoundToExactOperationAndCannotAdmitAnotherOperation() { + Fixture fixture = new Fixture(); + AssertionError fatal = new AssertionError("factory fatal"); + when(fixture.factory.acquire(any(), any(), anyDeadline())).thenThrow(fatal); + + assertThatThrownBy(fixture::execute).isSameAs(fatal); + assertThat(fatal.getSuppressed()).singleElement() + .isInstanceOf(RetainedCutoverReleaseRequiredException.class); + assertConflict(() -> fixture.coordinator.execute( + "operation-b", TARGET, fixture.password, TIMEOUT, MetadataMigrationProgressSink.NO_OP)); + + verify(fixture.factory).acquire(any(), any(), anyDeadline()); + verifyNoInteractions(fixture.provisioner, fixture.maintenance, fixture.executor); + } + + private static JdbcMetadataMigrationDeadline anyDeadline() { + return any(JdbcMetadataMigrationDeadline.class); + } + + private static TargetSchemaProvisioningException provisioningFailure() { + return new TargetSchemaProvisioningException( + MetadataDatabaseKind.MYSQL, + new TargetSchemaProvisioningFailure( + TargetSchemaProvisioningFailure.Phase.PRECONDITION, + "baseline", null, 0), + TargetSchemaConnectionDisposition.REUSABLE); + } + + private static void assertConflict(Runnable action) { + assertThatThrownBy(action::run).isInstanceOf(MigrationMaintenanceException.class); + } + + private static final class Fixture { + + private final Connection provisionConnection = mock(Connection.class); + private final Connection copyConnection = mock(Connection.class); + private final Connection sourceConnection = mock(Connection.class); + private final TargetJdbcConnectionFactory factory = mock(TargetJdbcConnectionFactory.class); + private final TargetJdbcConnectionLease provisionLease = mock(TargetJdbcConnectionLease.class); + private final TargetJdbcConnectionLease copyLease = mock(TargetJdbcConnectionLease.class); + private final FlywayTargetSchemaProvisioner provisioner = mock(FlywayTargetSchemaProvisioner.class); + private final MigrationMaintenanceOrchestrator maintenance = mock(MigrationMaintenanceOrchestrator.class); + private final MigrationMaintenanceLease maintenanceLease = mock(MigrationMaintenanceLease.class); + private final JdbcMetadataMigrationExecutor executor = mock(JdbcMetadataMigrationExecutor.class); + private final SecretValue password = mock(SecretValue.class); + private final RetainedCutoverCoordinator coordinator; + + private Fixture() { + when(provisionLease.targetIdentityHash()).thenReturn(IDENTITY); + when(copyLease.targetIdentityHash()).thenReturn(IDENTITY); + when(factory.acquire(same(TARGET), same(password), anyDeadline())) + .thenReturn(provisionLease, copyLease); + scopedTarget(provisionLease, provisionConnection); + scopedTarget(copyLease, copyConnection); + scopedSource(maintenanceLease, sourceConnection); + when(provisioner.provision(any(), any(), anyDeadline())) + .thenReturn(new TargetSchemaProvisioningOutcome( + TargetSchemaConnectionDisposition.REUSABLE)); + when(maintenance.acquire(eq(OPERATION_ID), any())).thenReturn(maintenanceLease); + coordinator = new RetainedCutoverCoordinator( + factory, provisioner, maintenance, executor, System::nanoTime); + } + + private RetainedCutoverResult execute() { + return coordinator.execute( + OPERATION_ID, TARGET, password, TIMEOUT, MetadataMigrationProgressSink.NO_OP); + } + + private static void scopedTarget(TargetJdbcConnectionLease lease, Connection connection) { + doAnswer(invocation -> { + TargetJdbcConnectionAction action = invocation.getArgument(0); + action.execute(connection); + return null; + }).when(lease).withConnection(any()); + } + + private static void scopedSource(MigrationMaintenanceLease lease, Connection connection) { + doAnswer(invocation -> { + MigrationSourceAction action = invocation.getArgument(0); + action.execute(connection); + return null; + }).when(lease).withSourceConnection(any()); + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverLifecycleTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverLifecycleTest.java new file mode 100644 index 0000000000..7bd8ee5549 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverLifecycleTest.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.same; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.time.Duration; +import java.util.Arrays; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceException; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceLease; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceOrchestrator; +import org.apache.hertzbeat.manager.maintenance.MigrationSourceAction; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +@Timeout(15) +class RetainedCutoverLifecycleTest { + + private static final String OPERATION_ID = "operation-a"; + private static final String IDENTITY = "a".repeat(64); + private static final Duration TIMEOUT = Duration.ofSeconds(1); + private static final MetadataDatabaseSettings TARGET = new MetadataDatabaseSettings( + MetadataDatabaseKind.MYSQL, "jdbc:mysql://db.example/hertzbeat", "migration"); + + @Test + void concurrentAndReentrantExecuteCannotReplaceTheOneActiveOperation() throws Exception { + Fixture fixture = new Fixture(); + CountDownLatch enteredProvision = new CountDownLatch(1); + CountDownLatch releaseProvision = new CountDownLatch(1); + doAnswer(invocation -> { + enteredProvision.countDown(); + await(releaseProvision); + return new TargetSchemaProvisioningOutcome(TargetSchemaConnectionDisposition.REUSABLE); + }).when(fixture.provisioner).provision(any(), any(), anyDeadline()); + ExecutorService callers = Executors.newSingleThreadExecutor(); + Future first = callers.submit(() -> fixture.execute()); + try { + assertThat(enteredProvision.await(1, TimeUnit.SECONDS)).isTrue(); + assertConflict(fixture::execute); + assertConflict(() -> fixture.coordinator.execute( + "operation-b", TARGET, fixture.password, TIMEOUT, + MetadataMigrationProgressSink.NO_OP)); + } finally { + releaseProvision.countDown(); + first.get(2, TimeUnit.SECONDS); + callers.shutdownNow(); + } + } + + @Test + void callbackLocalReentryFailsBeforeAnySecondConnectionOrMaintenanceMutation() { + Fixture fixture = new Fixture(); + AtomicReference reentrantFailure = new AtomicReference<>(); + doAnswer(invocation -> { + try { + fixture.execute(); + } catch (Throwable failure) { + reentrantFailure.set(failure); + } + return new TargetSchemaProvisioningOutcome(TargetSchemaConnectionDisposition.REUSABLE); + }).when(fixture.provisioner).provision(any(), any(), anyDeadline()); + + fixture.execute(); + + assertThat(reentrantFailure.get()).isInstanceOf(MigrationMaintenanceException.class); + verify(fixture.factory, never()).close(); + verify(fixture.executor, never()).close(); + } + + @Test + void borrowedSecretRemainsCallerOwnedAcrossSuccessAndFailure() { + Fixture fixture = new Fixture(); + try (SecretValue callerPassword = SecretValue.of("borrowed-password")) { + char[] before = callerPassword.copy(); + try { + when(fixture.factory.acquire(same(TARGET), same(callerPassword), anyDeadline())) + .thenReturn(fixture.provisionLease, fixture.copyLease); + fixture.execute(callerPassword); + assertThat(callerPassword.copy()).containsExactly(before); + fixture.coordinator.releaseRetained(OPERATION_ID); + when(fixture.factory.acquire(same(TARGET), same(callerPassword), anyDeadline())) + .thenThrow(new TargetJdbcConnectionException( + TargetJdbcConnectionErrorCode.UNAVAILABLE)); + assertThatThrownBy(() -> fixture.execute(callerPassword)) + .isInstanceOf(TargetJdbcConnectionException.class); + assertThat(callerPassword.copy()).containsExactly(before); + } finally { + Arrays.fill(before, '\0'); + } + } + } + + private static JdbcMetadataMigrationDeadline anyDeadline() { + return any(JdbcMetadataMigrationDeadline.class); + } + + private static void assertConflict(Runnable action) { + assertThatThrownBy(action::run).isInstanceOf(MigrationMaintenanceException.class); + } + + private static void await(CountDownLatch latch) { + boolean interrupted = false; + try { + while (true) { + try { + latch.await(); + return; + } catch (InterruptedException ignored) { + interrupted = true; + } + } + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + + private static final class Fixture { + + private final Connection provisionConnection = mock(Connection.class); + private final Connection copyConnection = mock(Connection.class); + private final Connection sourceConnection = mock(Connection.class); + private final TargetJdbcConnectionFactory factory = mock(TargetJdbcConnectionFactory.class); + private final TargetJdbcConnectionLease provisionLease = mock(TargetJdbcConnectionLease.class); + private final TargetJdbcConnectionLease copyLease = mock(TargetJdbcConnectionLease.class); + private final FlywayTargetSchemaProvisioner provisioner = mock(FlywayTargetSchemaProvisioner.class); + private final MigrationMaintenanceOrchestrator maintenance = mock(MigrationMaintenanceOrchestrator.class); + private final MigrationMaintenanceLease maintenanceLease = mock(MigrationMaintenanceLease.class); + private final JdbcMetadataMigrationExecutor executor = mock(JdbcMetadataMigrationExecutor.class); + private final SecretValue password = mock(SecretValue.class); + private final RetainedCutoverCoordinator coordinator; + + private Fixture() { + when(provisionLease.targetIdentityHash()).thenReturn(IDENTITY); + when(copyLease.targetIdentityHash()).thenReturn(IDENTITY); + when(factory.acquire(same(TARGET), same(password), anyDeadline())) + .thenReturn(provisionLease, copyLease); + scopedTarget(provisionLease, provisionConnection); + scopedTarget(copyLease, copyConnection); + scopedSource(maintenanceLease, sourceConnection); + when(provisioner.provision(any(), any(), anyDeadline())) + .thenReturn(new TargetSchemaProvisioningOutcome( + TargetSchemaConnectionDisposition.REUSABLE)); + when(maintenance.acquire(eq(OPERATION_ID), any())).thenReturn(maintenanceLease); + coordinator = new RetainedCutoverCoordinator( + factory, provisioner, maintenance, executor, System::nanoTime); + } + + private RetainedCutoverResult execute() { + return execute(password); + } + + private RetainedCutoverResult execute(SecretValue borrowedPassword) { + return coordinator.execute( + OPERATION_ID, TARGET, borrowedPassword, TIMEOUT, MetadataMigrationProgressSink.NO_OP); + } + + private static void scopedTarget(TargetJdbcConnectionLease lease, Connection connection) { + doAnswer(invocation -> { + TargetJdbcConnectionAction action = invocation.getArgument(0); + action.execute(connection); + return null; + }).when(lease).withConnection(any()); + } + + private static void scopedSource(MigrationMaintenanceLease lease, Connection connection) { + doAnswer(invocation -> { + MigrationSourceAction action = invocation.getArgument(0); + action.execute(connection); + return null; + }).when(lease).withSourceConnection(any()); + } + } +} From 9ba2f5cd17f208e8c2b9a9d29d24f7bb364bc1f4 Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 13:37:25 +0800 Subject: [PATCH 51/71] Prepare metadata cutovers durably --- .../workflow/RetainedCutoverCoordinator.java | 17 +- .../workflow/RetainedCutoverPreparation.java | 24 ++ .../RetainedCutoverPreparationContext.java | 28 ++ .../setup/workflow/RetainedCutoverSteps.java | 21 ++ .../RetainedCutoverCoordinatorTest.java | 6 +- .../RetainedCutoverFactoryRaceTest.java | 3 +- .../workflow/RetainedCutoverFailureTest.java | 9 +- .../RetainedCutoverLifecycleTest.java | 5 +- .../RetainedCutoverPreparationTest.java | 297 ++++++++++++++++++ 9 files changed, 399 insertions(+), 11 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverPreparation.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverPreparationContext.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverPreparationTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java index 0fd7adc442..ae9ebac510 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java @@ -48,13 +48,22 @@ final class RetainedCutoverCoordinator { MetadataDatabaseSettings target, SecretValue borrowedPassword, Duration timeout, - MetadataMigrationProgressSink progress) { - requireRequest(operationId, target, borrowedPassword, timeout, progress); + MetadataMigrationProgressSink progress, + RetainedCutoverPreparation preparation) { + requireRequest(operationId, target, borrowedPassword, timeout, progress, preparation); JdbcMetadataMigrationDeadline deadline = JdbcMetadataMigrationDeadline.start(timeout, ticker); RetainedCutoverState.Execution execution = state.reserve(operationId); TargetJdbcConnectionLease provisionLease = acquire(execution, target, borrowedPassword, deadline); String provisionIdentity = targetIdentity(execution, provisionLease, deadline); execution.targetIdentityHash(provisionIdentity); + RetainedCutoverOutcome preparationOutcome = steps.prepare( + preparation, + new RetainedCutoverPreparationContext(operationId, provisionIdentity), + deadline); + if (!preparationOutcome.successful()) { + return finish(execution, RetainedCutoverRelease.resources( + provisionLease, null, preparationOutcome, false), deadline); + } RetainedCutoverOutcome provisionOutcome = steps.provision( provisionLease, target, deadline); if (!provisionOutcome.successful()) { @@ -207,12 +216,14 @@ final class RetainedCutoverCoordinator { MetadataDatabaseSettings target, SecretValue password, Duration timeout, - MetadataMigrationProgressSink progress) { + MetadataMigrationProgressSink progress, + RetainedCutoverPreparation preparation) { requireOperationId(operationId); Objects.requireNonNull(target, "target"); Objects.requireNonNull(password, "password"); Objects.requireNonNull(timeout, "timeout"); Objects.requireNonNull(progress, "progress"); + Objects.requireNonNull(preparation, "preparation"); } private static void requireOperationId(String operationId) { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverPreparation.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverPreparation.java new file mode 100644 index 0000000000..2b7432430e --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverPreparation.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +/** + * Persists the secret-free preparation boundary before target schema provisioning begins. + * + *

The callback is synchronous and must not retain its context. A normal return means the + * operation is durably prepared for the exact target identity. Callers must pass this seam + * explicitly; {@link #NO_OP} exists only for isolated tests that do not exercise durable workflow + * state. + */ +@FunctionalInterface +interface RetainedCutoverPreparation { + + RetainedCutoverPreparation NO_OP = context -> { }; + + void prepare(RetainedCutoverPreparationContext context); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverPreparationContext.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverPreparationContext.java new file mode 100644 index 0000000000..a1ff1b82cb --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverPreparationContext.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; +import java.util.regex.Pattern; +import org.apache.hertzbeat.manager.setup.api.OperationIdValidator; + +/** Secret-free identity required to durably prepare one retained cutover. */ +record RetainedCutoverPreparationContext(String operationId, String targetIdentityHash) { + + private static final Pattern IDENTITY_HASH = Pattern.compile("[0-9a-f]{64}"); + + RetainedCutoverPreparationContext { + if (!OperationIdValidator.isSafe(operationId)) { + throw new IllegalArgumentException("Unsafe operation identifier"); + } + Objects.requireNonNull(targetIdentityHash, "targetIdentityHash"); + if (!IDENTITY_HASH.matcher(targetIdentityHash).matches()) { + throw new IllegalArgumentException("Invalid target identity hash"); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverSteps.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverSteps.java index b8a8ec0cee..f4faa0193d 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverSteps.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverSteps.java @@ -40,6 +40,27 @@ final class RetainedCutoverSteps { return targetFactory.acquire(target, password, deadline); } + RetainedCutoverOutcome prepare( + RetainedCutoverPreparation preparation, + RetainedCutoverPreparationContext context, + JdbcMetadataMigrationDeadline deadline) { + try { + requirePreparationBudget(deadline); + preparation.prepare(context); + requirePreparationBudget(deadline); + return RetainedCutoverOutcome.success(); + } catch (RuntimeException | Error failure) { + return RetainedCutoverOutcome.failure(failure); + } + } + + private static void requirePreparationBudget(JdbcMetadataMigrationDeadline deadline) { + if (Thread.currentThread().isInterrupted()) { + throw new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT); + } + deadline.remainingDuration(); + } + RetainedCutoverOutcome provision( TargetJdbcConnectionLease lease, MetadataDatabaseSettings target, diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinatorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinatorTest.java index 76f6d26396..82f27e87a0 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinatorTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinatorTest.java @@ -146,7 +146,8 @@ class RetainedCutoverCoordinatorTest { verify(fixture.executor).execute(any(), any(), any(), anyDeadline(), any()); assertConflict(fixture::execute); assertConflict(() -> fixture.coordinator.execute( - "operation-b", TARGET, fixture.password, TIMEOUT, MetadataMigrationProgressSink.NO_OP)); + "operation-b", TARGET, fixture.password, TIMEOUT, + MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP)); } @Test @@ -321,7 +322,8 @@ class RetainedCutoverCoordinatorTest { private RetainedCutoverResult execute() { return coordinator.execute( - OPERATION_ID, TARGET, password, TIMEOUT, MetadataMigrationProgressSink.NO_OP); + OPERATION_ID, TARGET, password, TIMEOUT, + MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP); } private static void scopedTarget(TargetJdbcConnectionLease lease, Connection connection) { diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverFactoryRaceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverFactoryRaceTest.java index 1f54b78299..4df56d54b7 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverFactoryRaceTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverFactoryRaceTest.java @@ -119,7 +119,8 @@ class RetainedCutoverFactoryRaceTest { private static RetainedCutoverResult execute( RetainedCutoverCoordinator coordinator, String operationId, SecretValue password) { return coordinator.execute( - operationId, TARGET, password, TIMEOUT, MetadataMigrationProgressSink.NO_OP); + operationId, TARGET, password, TIMEOUT, + MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP); } private static void assertConflict(Runnable action) { diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverFailureTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverFailureTest.java index 175c7582e6..179cfbadd3 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverFailureTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverFailureTest.java @@ -168,7 +168,8 @@ class RetainedCutoverFailureTest { TARGET, fixture.password, Duration.ZERO, - MetadataMigrationProgressSink.NO_OP)) + MetadataMigrationProgressSink.NO_OP, + RetainedCutoverPreparation.NO_OP)) .isInstanceOf(MetadataMigrationException.class); assertThat(fixture.execute().status()).isEqualTo(RetainedCutoverResult.Status.RETAINED_SUCCESS); @@ -210,7 +211,8 @@ class RetainedCutoverFailureTest { assertThat(fatal.getSuppressed()).singleElement() .isInstanceOf(RetainedCutoverReleaseRequiredException.class); assertConflict(() -> fixture.coordinator.execute( - "operation-b", TARGET, fixture.password, TIMEOUT, MetadataMigrationProgressSink.NO_OP)); + "operation-b", TARGET, fixture.password, TIMEOUT, + MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP)); verify(fixture.factory).acquire(any(), any(), anyDeadline()); verifyNoInteractions(fixture.provisioner, fixture.maintenance, fixture.executor); @@ -266,7 +268,8 @@ class RetainedCutoverFailureTest { private RetainedCutoverResult execute() { return coordinator.execute( - OPERATION_ID, TARGET, password, TIMEOUT, MetadataMigrationProgressSink.NO_OP); + OPERATION_ID, TARGET, password, TIMEOUT, + MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP); } private static void scopedTarget(TargetJdbcConnectionLease lease, Connection connection) { diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverLifecycleTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverLifecycleTest.java index 7bd8ee5549..127750976c 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverLifecycleTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverLifecycleTest.java @@ -63,7 +63,7 @@ class RetainedCutoverLifecycleTest { assertConflict(fixture::execute); assertConflict(() -> fixture.coordinator.execute( "operation-b", TARGET, fixture.password, TIMEOUT, - MetadataMigrationProgressSink.NO_OP)); + MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP)); } finally { releaseProvision.countDown(); first.get(2, TimeUnit.SECONDS); @@ -177,7 +177,8 @@ class RetainedCutoverLifecycleTest { private RetainedCutoverResult execute(SecretValue borrowedPassword) { return coordinator.execute( - OPERATION_ID, TARGET, borrowedPassword, TIMEOUT, MetadataMigrationProgressSink.NO_OP); + OPERATION_ID, TARGET, borrowedPassword, TIMEOUT, + MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP); } private static void scopedTarget(TargetJdbcConnectionLease lease, Connection connection) { diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverPreparationTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverPreparationTest.java new file mode 100644 index 0000000000..6d30b2a066 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverPreparationTest.java @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.same; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceException; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceLease; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceOrchestrator; +import org.apache.hertzbeat.manager.maintenance.MigrationSourceAction; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.mockito.InOrder; + +@Timeout(15) +class RetainedCutoverPreparationTest { + + private static final String OPERATION = "operation-a"; + private static final String IDENTITY = "a".repeat(64); + private static final Duration TIMEOUT = Duration.ofNanos(100); + private static final MetadataDatabaseSettings TARGET = new MetadataDatabaseSettings( + MetadataDatabaseKind.MYSQL, "jdbc:mysql://db.example/hertzbeat", "migration"); + + @Test + void preparesExactlyOnceAfterFirstIdentityAndBeforeAnyProvisionMutation() { + Fixture fixture = new Fixture(); + AtomicReference observed = new AtomicReference<>(); + RetainedCutoverPreparation preparation = mock(RetainedCutoverPreparation.class); + doAnswer(invocation -> { + observed.set(invocation.getArgument(0)); + return null; + }).when(preparation).prepare(any()); + + fixture.execute(preparation); + + assertThat(observed.get()).isEqualTo( + new RetainedCutoverPreparationContext(OPERATION, IDENTITY)); + InOrder order = inOrder(fixture.factory, fixture.provisionLease, preparation, fixture.provisioner); + order.verify(fixture.factory).acquire(same(TARGET), same(fixture.password), anyDeadline()); + order.verify(fixture.provisionLease).targetIdentityHash(); + order.verify(preparation).prepare(observed.get()); + order.verify(fixture.provisionLease).withConnection(any()); + order.verify(fixture.provisioner).provision( + same(fixture.provisionConnection), eq(MetadataDatabaseKind.MYSQL), anyDeadline()); + verify(preparation).prepare(any()); + } + + @Test + void elapsedRootDeadlineAfterPreparationPreventsProvision() { + Fixture fixture = new Fixture(); + RetainedCutoverPreparation preparation = context -> fixture.ticker.set(100); + + assertThatThrownBy(() -> fixture.execute(preparation)) + .isInstanceOfSatisfying(MetadataMigrationException.class, failure -> + assertThat(failure.code()).isEqualTo(MetadataMigrationErrorCode.TIMEOUT)); + + verify(fixture.provisionLease).close(); + verifyNoInteractions(fixture.provisioner, fixture.maintenance, fixture.executor); + } + + @Test + void expiredRootDeadlineAfterFirstIdentityNeverInvokesPreparation() { + Fixture fixture = new Fixture(); + RetainedCutoverPreparation preparation = mock(RetainedCutoverPreparation.class); + when(fixture.provisionLease.targetIdentityHash()).thenAnswer(invocation -> { + fixture.ticker.set(100); + return IDENTITY; + }); + + assertThatThrownBy(() -> fixture.execute(preparation)) + .isInstanceOfSatisfying(MetadataMigrationException.class, failure -> + assertThat(failure.code()).isEqualTo(MetadataMigrationErrorCode.TIMEOUT)); + + verifyNoInteractions(preparation, fixture.provisioner, fixture.maintenance, fixture.executor); + verify(fixture.provisionLease).close(); + } + + @Test + void interruptAfterFirstIdentityNeverInvokesPreparationAndIsRestored() { + Fixture fixture = new Fixture(); + RetainedCutoverPreparation preparation = mock(RetainedCutoverPreparation.class); + when(fixture.provisionLease.targetIdentityHash()).thenAnswer(invocation -> { + Thread.currentThread().interrupt(); + return IDENTITY; + }); + doAnswer(invocation -> { + assertThat(Thread.currentThread().isInterrupted()).isFalse(); + return null; + }).when(fixture.provisionLease).close(); + + try { + assertThatThrownBy(() -> fixture.execute(preparation)) + .isInstanceOfSatisfying(MetadataMigrationException.class, failure -> + assertThat(failure.code()).isEqualTo(MetadataMigrationErrorCode.TIMEOUT)); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } finally { + Thread.interrupted(); + } + verifyNoInteractions(preparation, fixture.provisioner, fixture.maintenance, fixture.executor); + verify(fixture.provisionLease).close(); + } + + @Test + void stableAndUnexpectedPreparationFailuresCloseFirstLeaseWithoutProvision() { + Fixture stable = new Fixture(); + MetadataMigrationException expected = new MetadataMigrationException( + MetadataMigrationErrorCode.VERIFICATION); + RetainedCutoverPreparation stableFailure = context -> { + throw expected; + }; + + assertThatThrownBy(() -> stable.execute(stableFailure)).isSameAs(expected); + verify(stable.provisionLease).close(); + verifyNoInteractions(stable.provisioner, stable.maintenance, stable.executor); + + Fixture unexpected = new Fixture(); + RetainedCutoverPreparation privateFailure = context -> { + throw new IllegalStateException("private preparation diagnostic"); + }; + assertThatThrownBy(() -> unexpected.execute(privateFailure)) + .isInstanceOfSatisfying(RetainedCutoverException.class, failure -> + assertThat(failure.code()).isEqualTo(RetainedCutoverErrorCode.EXECUTION_FAILED)) + .hasNoCause(); + verify(unexpected.provisionLease).close(); + verifyNoInteractions(unexpected.provisioner, unexpected.maintenance, unexpected.executor); + } + + @Test + void fatalPreparationRemainsPrimaryAcrossExactCloseRetry() { + Fixture fixture = new Fixture(); + AssertionError fatal = new AssertionError("preparation fatal"); + RetainedCutoverPreparation preparation = mock(RetainedCutoverPreparation.class); + doThrow(fatal).when(preparation).prepare(any()); + doThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED)) + .doNothing().when(fixture.provisionLease).close(); + + assertThatThrownBy(() -> fixture.execute(preparation)).isSameAs(fatal); + assertThat(fatal.getSuppressed()).singleElement() + .isInstanceOf(RetainedCutoverReleaseRequiredException.class); + assertConflict(() -> fixture.coordinator.retryRelease("operation-b", Duration.ofSeconds(1))); + assertThatThrownBy(() -> fixture.coordinator.retryRelease(OPERATION, Duration.ofSeconds(1))) + .isSameAs(fatal); + + verify(preparation).prepare(any()); + verify(fixture.provisionLease, times(2)).close(); + verifyNoInteractions(fixture.provisioner, fixture.maintenance, fixture.executor); + } + + @Test + void preparationInterruptIsClearedForCloseAndRestoredForCaller() { + Fixture fixture = new Fixture(); + RetainedCutoverPreparation preparation = context -> Thread.currentThread().interrupt(); + doAnswer(invocation -> { + assertThat(Thread.currentThread().isInterrupted()).isFalse(); + return null; + }).when(fixture.provisionLease).close(); + + try { + assertThatThrownBy(() -> fixture.execute(preparation)) + .isInstanceOf(MetadataMigrationException.class); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } finally { + Thread.interrupted(); + } + verifyNoInteractions(fixture.provisioner, fixture.maintenance, fixture.executor); + } + + @Test + void preparationReentryAndForeignOperationConflictBeforeProvision() { + Fixture fixture = new Fixture(); + AtomicReference sameOperation = new AtomicReference<>(); + AtomicReference foreignOperation = new AtomicReference<>(); + RetainedCutoverPreparation preparation = context -> { + capture(sameOperation, () -> fixture.execute(RetainedCutoverPreparation.NO_OP)); + capture(foreignOperation, () -> fixture.coordinator.execute( + "operation-b", TARGET, fixture.password, TIMEOUT, + MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP)); + }; + + fixture.execute(preparation); + + assertThat(sameOperation.get()).isInstanceOf(MigrationMaintenanceException.class); + assertThat(foreignOperation.get()).isInstanceOf(MigrationMaintenanceException.class); + verify(fixture.provisioner).provision(any(), any(), anyDeadline()); + } + + @Test + void contextRejectsUnsafeIdentityAndExposesNoCredentialSurface() { + assertThatThrownBy(() -> new RetainedCutoverPreparationContext("../operation", IDENTITY)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> new RetainedCutoverPreparationContext(OPERATION, "invalid")) + .isInstanceOf(IllegalArgumentException.class); + + RetainedCutoverPreparationContext context = + new RetainedCutoverPreparationContext(OPERATION, IDENTITY); + assertThat(context.getClass().getRecordComponents()) + .extracting(component -> component.getName()) + .containsExactly("operationId", "targetIdentityHash"); + assertThat(context.toString()) + .doesNotContain("jdbc:", "password", "username", "table", "checksum"); + } + + private static JdbcMetadataMigrationDeadline anyDeadline() { + return any(JdbcMetadataMigrationDeadline.class); + } + + private static void capture(AtomicReference target, Runnable action) { + try { + action.run(); + } catch (Throwable failure) { + target.set(failure); + } + } + + private static void assertConflict(Runnable action) { + assertThatThrownBy(action::run).isInstanceOf(MigrationMaintenanceException.class); + } + + private static final class Fixture { + + private final Connection provisionConnection = mock(Connection.class); + private final Connection copyConnection = mock(Connection.class); + private final Connection sourceConnection = mock(Connection.class); + private final TargetJdbcConnectionFactory factory = mock(TargetJdbcConnectionFactory.class); + private final TargetJdbcConnectionLease provisionLease = mock(TargetJdbcConnectionLease.class); + private final TargetJdbcConnectionLease copyLease = mock(TargetJdbcConnectionLease.class); + private final FlywayTargetSchemaProvisioner provisioner = mock(FlywayTargetSchemaProvisioner.class); + private final MigrationMaintenanceOrchestrator maintenance = mock(MigrationMaintenanceOrchestrator.class); + private final MigrationMaintenanceLease maintenanceLease = mock(MigrationMaintenanceLease.class); + private final JdbcMetadataMigrationExecutor executor = mock(JdbcMetadataMigrationExecutor.class); + private final SecretValue password = mock(SecretValue.class); + private final AtomicLong ticker = new AtomicLong(); + private final RetainedCutoverCoordinator coordinator; + + private Fixture() { + when(provisionLease.targetIdentityHash()).thenReturn(IDENTITY); + when(copyLease.targetIdentityHash()).thenReturn(IDENTITY); + when(factory.acquire(same(TARGET), same(password), anyDeadline())) + .thenReturn(provisionLease, copyLease); + scopedTarget(provisionLease, provisionConnection); + scopedTarget(copyLease, copyConnection); + scopedSource(maintenanceLease, sourceConnection); + when(provisioner.provision(any(), any(), anyDeadline())) + .thenReturn(new TargetSchemaProvisioningOutcome( + TargetSchemaConnectionDisposition.REUSABLE)); + when(maintenance.acquire(eq(OPERATION), any())).thenReturn(maintenanceLease); + coordinator = new RetainedCutoverCoordinator( + factory, provisioner, maintenance, executor, ticker::get); + } + + private RetainedCutoverResult execute(RetainedCutoverPreparation preparation) { + return coordinator.execute( + OPERATION, TARGET, password, TIMEOUT, + MetadataMigrationProgressSink.NO_OP, preparation); + } + + private static void scopedTarget(TargetJdbcConnectionLease lease, Connection connection) { + doAnswer(invocation -> { + TargetJdbcConnectionAction action = invocation.getArgument(0); + action.execute(connection); + return null; + }).when(lease).withConnection(any()); + } + + private static void scopedSource(MigrationMaintenanceLease lease, Connection connection) { + doAnswer(invocation -> { + MigrationSourceAction action = invocation.getArgument(0); + action.execute(connection); + return null; + }).when(lease).withSourceConnection(any()); + } + } +} From bb3aef3ecf8d2563b4cdb600d79ea3e4e61dd2ef Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 13:53:28 +0800 Subject: [PATCH 52/71] Confirm metadata operation journal writes --- .../workflow/FileMigrationOperationStore.java | 84 ++++++ .../FileMigrationOperationStoreExactTest.java | 241 ++++++++++++++++++ 2 files changed, 325 insertions(+) create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStoreExactTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java index ca05277d90..171a5d7c0a 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java @@ -67,6 +67,32 @@ public final class FileMigrationOperationStore implements MigrationOperationStor }); } + /** Creates or confirms one fully equal PENDING snapshot under the store lock. */ + MigrationOperationSnapshot createOrConfirm(MigrationOperationSnapshot snapshot) { + Objects.requireNonNull(snapshot, "snapshot"); + if (snapshot.state() != MigrationOperationState.PENDING) { + throw failure(SetupErrorCode.INVALID_REQUEST); + } + return locked(() -> { + List snapshots = read(); + for (MigrationOperationSnapshot current : snapshots) { + if (current.operationId().equals(snapshot.operationId())) { + if (current.equals(snapshot)) { + writeAndConfirm(snapshots); + return snapshot; + } + throw failure(SetupErrorCode.OPERATION_CONFLICT); + } + if (!current.terminal()) { + throw failure(SetupErrorCode.OPERATION_CONFLICT); + } + } + snapshots.add(snapshot); + writeAndConfirm(snapshots); + return snapshot; + }); + } + @Override public Optional find(String operationId) { requireSafeId(operationId); @@ -88,6 +114,15 @@ public final class FileMigrationOperationStore implements MigrationOperationStor return locked(() -> transition(read(), operationId, expectedState, replacement)); } + /** Transitions or confirms one fully equal replacement under the store lock. */ + MigrationOperationSnapshot compareAndTransitionOrConfirm( + String operationId, MigrationOperationState expectedState, MigrationOperationSnapshot replacement) { + requireSafeId(operationId); + Objects.requireNonNull(expectedState, "expectedState"); + Objects.requireNonNull(replacement, "replacement"); + return locked(() -> transitionOrConfirm(read(), operationId, expectedState, replacement)); + } + private MigrationOperationSnapshot transition( List snapshots, String operationId, MigrationOperationState expectedState, MigrationOperationSnapshot replacement) { @@ -107,6 +142,29 @@ public final class FileMigrationOperationStore implements MigrationOperationStor throw failure(SetupErrorCode.OPERATION_NOT_FOUND); } + private MigrationOperationSnapshot transitionOrConfirm( + List snapshots, String operationId, + MigrationOperationState expectedState, MigrationOperationSnapshot replacement) { + for (int index = 0; index < snapshots.size(); index++) { + MigrationOperationSnapshot current = snapshots.get(index); + if (current.operationId().equals(operationId)) { + if (current.equals(replacement)) { + writeAndConfirm(snapshots); + return replacement; + } + if (current.state() != expectedState) { + throw failure(SetupErrorCode.OPERATION_CONFLICT); + } + transitionPolicy.requireAllowed(current, replacement); + snapshots.set(index, replacement); + trim(snapshots); + writeAndConfirm(snapshots); + return replacement; + } + } + throw failure(SetupErrorCode.OPERATION_NOT_FOUND); + } + private List read() { if (!Files.exists(operationFile, LinkOption.NOFOLLOW_LINKS)) { return new ArrayList<>(); @@ -139,6 +197,32 @@ public final class FileMigrationOperationStore implements MigrationOperationStor } } + private void writeAndConfirm(List snapshots) { + collectionPolicy.validate(snapshots); + byte[] encoded = codec.encode(snapshots); + try { + publisher.publish(operationFile, encoded); + } catch (CommittedSetupFileDurabilityException uncertain) { + confirmAndRepublish(snapshots, encoded); + } catch (IOException failure) { + throw failure(SetupErrorCode.CONFIG_WRITE_FAILED); + } finally { + Arrays.fill(encoded, (byte) 0); + } + } + + private void confirmAndRepublish(List intended, byte[] encoded) { + List persisted = read(); + if (!persisted.equals(intended)) { + throw failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + try { + publisher.publish(operationFile, encoded); + } catch (IOException failure) { + throw failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + } + private void trim(List snapshots) { while (snapshots.stream().filter(MigrationOperationSnapshot::terminal).count() > HISTORY_LIMIT) { int oldestTerminal = -1; diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStoreExactTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStoreExactTest.java new file mode 100644 index 0000000000..3355162d7f --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStoreExactTest.java @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.security.CommittedSetupFileDurabilityException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class FileMigrationOperationStoreExactTest { + + private static final String IDENTITY = "a".repeat(64); + private static final String GENERATION = "candidate-generation"; + private static final Instant CREATED = Instant.parse("2026-08-10T01:00:00Z"); + + @TempDir + private Path root; + + @Test + void exactCreateAndTransitionRetriesAreIdempotent() { + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + MigrationOperationSnapshot pending = pending("operation-a", IDENTITY, GENERATION); + MigrationOperationSnapshot running = running(pending); + + assertThat(store.createOrConfirm(pending)).isEqualTo(pending); + assertThat(store.createOrConfirm(pending)).isEqualTo(pending); + assertThat(store.compareAndTransitionOrConfirm( + pending.operationId(), MigrationOperationState.PENDING, running)).isEqualTo(running); + assertThat(store.compareAndTransitionOrConfirm( + pending.operationId(), MigrationOperationState.PENDING, running)).isEqualTo(running); + } + + @Test + void rejectsDifferingIdentityAndGeneration() { + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + MigrationOperationSnapshot pending = pending("operation-a", IDENTITY, GENERATION); + store.createOrConfirm(pending); + + assertStoreError(SetupErrorCode.OPERATION_CONFLICT, + () -> store.createOrConfirm(pending("operation-a", "b".repeat(64), GENERATION))); + assertStoreError(SetupErrorCode.OPERATION_CONFLICT, + () -> store.createOrConfirm(pending("operation-a", IDENTITY, "other-generation"))); + } + + @Test + void differentActiveOperationAndAdvancedNonExactStateRemainConflicts() { + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + MigrationOperationSnapshot pending = pending("operation-a", IDENTITY, GENERATION); + store.createOrConfirm(pending); + + assertStoreError(SetupErrorCode.OPERATION_CONFLICT, + () -> store.createOrConfirm(pending("operation-b", IDENTITY, GENERATION))); + + MigrationOperationSnapshot running = running(pending); + store.compareAndTransitionOrConfirm(pending.operationId(), MigrationOperationState.PENDING, running); + MigrationOperationSnapshot later = new MigrationOperationSnapshot( + running.operationId(), running.state(), running.target(), running.applyMode(), running.stage(), + 20, running.createdAt(), running.startedAt(), running.completedAt(), running.verificationState(), + running.errorCode(), running.rollbackOrigin(), running.nextPollAfterMillis(), + running.activationAvailable(), running.restartRequired(), running.externalApplyRequired(), + running.targetIdentityHash(), running.managedCandidateGeneration()); + assertStoreError(SetupErrorCode.OPERATION_CONFLICT, () -> store.compareAndTransitionOrConfirm( + pending.operationId(), MigrationOperationState.PENDING, later)); + } + + @Test + void committedCreateIsConfirmedByAuthoritativeReadBack() { + MigrationOperationSnapshot pending = pending("operation-a", IDENTITY, GENERATION); + MigrationOperationFilePublisher committed = new MigrationOperationFilePublisher(root); + AtomicBoolean first = new AtomicBoolean(true); + FileMigrationOperationStore uncertain = new FileMigrationOperationStore(root, (target, content) -> { + committed.publish(target, content); + if (first.getAndSet(false)) { + throw new CommittedSetupFileDurabilityException(); + } + }); + + assertThat(uncertain.createOrConfirm(pending)).isEqualTo(pending); + } + + @Test + void committedTransitionIsConfirmedByAuthoritativeReadBack() { + MigrationOperationSnapshot pending = pending("operation-a", IDENTITY, GENERATION); + new FileMigrationOperationStore(root).create(pending); + MigrationOperationSnapshot running = running(pending); + MigrationOperationFilePublisher committed = new MigrationOperationFilePublisher(root); + AtomicBoolean first = new AtomicBoolean(true); + FileMigrationOperationStore uncertain = new FileMigrationOperationStore(root, (target, content) -> { + committed.publish(target, content); + if (first.getAndSet(false)) { + throw new CommittedSetupFileDurabilityException(); + } + }); + + assertThat(uncertain.compareAndTransitionOrConfirm( + pending.operationId(), MigrationOperationState.PENDING, running)).isEqualTo(running); + } + + @Test + void exactReplayMustConfirmDurabilityBeforeReturningSuccess() { + MigrationOperationSnapshot pending = pending("operation-a", IDENTITY, GENERATION); + MigrationOperationFilePublisher committed = new MigrationOperationFilePublisher(root); + AtomicInteger publications = new AtomicInteger(); + FileMigrationOperationStore uncertain = new FileMigrationOperationStore(root, (target, content) -> { + committed.publish(target, content); + if (publications.incrementAndGet() <= 4) { + throw new CommittedSetupFileDurabilityException(); + } + }); + + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> uncertain.createOrConfirm(pending)); + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> uncertain.createOrConfirm(pending)); + assertThat(uncertain.createOrConfirm(pending)).isEqualTo(pending); + assertThat(publications).hasValue(5); + } + + @Test + void exactTransitionReplayMustConfirmDurabilityBeforeReturningSuccess() { + MigrationOperationSnapshot pending = pending("operation-a", IDENTITY, GENERATION); + MigrationOperationSnapshot running = running(pending); + new FileMigrationOperationStore(root).create(pending); + MigrationOperationFilePublisher committed = new MigrationOperationFilePublisher(root); + AtomicInteger publications = new AtomicInteger(); + FileMigrationOperationStore uncertain = new FileMigrationOperationStore(root, (target, content) -> { + committed.publish(target, content); + if (publications.incrementAndGet() <= 4) { + throw new CommittedSetupFileDurabilityException(); + } + }); + + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> uncertain.compareAndTransitionOrConfirm( + pending.operationId(), MigrationOperationState.PENDING, running)); + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> uncertain.compareAndTransitionOrConfirm( + pending.operationId(), MigrationOperationState.PENDING, running)); + assertThat(uncertain.compareAndTransitionOrConfirm( + pending.operationId(), MigrationOperationState.PENDING, running)).isEqualTo(running); + assertThat(publications).hasValue(5); + } + + @Test + void uncertainCreateMissingOrCorruptFailsClosed() { + MigrationOperationSnapshot pending = pending("operation-a", IDENTITY, GENERATION); + FileMigrationOperationStore missing = new FileMigrationOperationStore(root, (target, content) -> { + throw new CommittedSetupFileDurabilityException(); + }); + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> missing.createOrConfirm(pending)); + + FileMigrationOperationStore corrupt = new FileMigrationOperationStore(root, (target, content) -> { + Files.createDirectories(target.getParent()); + Files.writeString(target, "schema=99\n", StandardCharsets.UTF_8); + throw new CommittedSetupFileDurabilityException(); + }); + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> corrupt.createOrConfirm(pending)); + } + + @Test + void uncertainTransitionMissingOrCorruptFailsClosed() { + MigrationOperationSnapshot pending = pending("operation-a", IDENTITY, GENERATION); + MigrationOperationSnapshot running = running(pending); + new FileMigrationOperationStore(root).create(pending); + FileMigrationOperationStore missing = new FileMigrationOperationStore(root, (target, content) -> { + Files.delete(target); + throw new CommittedSetupFileDurabilityException(); + }); + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> missing.compareAndTransitionOrConfirm( + pending.operationId(), MigrationOperationState.PENDING, running)); + + new FileMigrationOperationStore(root).create(pending); + FileMigrationOperationStore corrupt = new FileMigrationOperationStore(root, (target, content) -> { + Files.writeString(target, "schema=99\n", StandardCharsets.UTF_8); + throw new CommittedSetupFileDurabilityException(); + }); + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> corrupt.compareAndTransitionOrConfirm( + pending.operationId(), MigrationOperationState.PENDING, running)); + } + + @Test + void exactMethodSurfaceContainsNoMigrationPayloadOrCredentialFields() { + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + assertThat(store.toString()) + .doesNotContain("jdbc:", "password", "username", IDENTITY, GENERATION); + } + + private static MigrationOperationSnapshot pending(String operation, String identity, String generation) { + return new MigrationOperationSnapshot(operation, MigrationOperationState.PENDING, MigrationTarget.MYSQL, + ApplyMode.MANAGED_WRITE, MigrationStage.QUEUED, 0, CREATED, null, null, + VerificationState.PENDING, null, null, 1000, false, false, false, + identity, generation); + } + + private static MigrationOperationSnapshot running(MigrationOperationSnapshot pending) { + return new MigrationOperationSnapshot( + pending.operationId(), MigrationOperationState.RUNNING, pending.target(), pending.applyMode(), + MigrationStage.COPYING, 10, pending.createdAt(), pending.createdAt().plusSeconds(1), null, + VerificationState.PENDING, null, null, 1000, false, false, false, + pending.targetIdentityHash(), pending.managedCandidateGeneration()); + } + + private static void assertStoreError(SetupErrorCode code, ThrowingAction action) { + assertThatThrownBy(action::run) + .isInstanceOfSatisfying(MigrationOperationStoreException.class, + failure -> assertThat(failure.errorCode()).isEqualTo(code)) + .hasNoCause() + .hasMessageNotContaining("jdbc") + .hasMessageNotContaining("password") + .hasMessageNotContaining("username") + .hasMessageNotContaining("schema=99"); + } + + @FunctionalInterface + private interface ThrowingAction { + void run() throws Exception; + } +} From 5fb86ee9b075358b4be3ab29d3a59a4adca23274 Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 14:11:33 +0800 Subject: [PATCH 53/71] Confirm migration candidate durability --- .../config/MigrationCandidateFileIo.java | 21 ++ .../setup/config/MigrationCandidateStore.java | 64 ++-- .../SecureMigrationCandidateFileIo.java | 66 ++++ .../MigrationCandidateDurabilityTest.java | 316 ++++++++++++++++++ .../CommittedAtomicReplaceTestSupport.java | 24 ++ 5 files changed, 469 insertions(+), 22 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateFileIo.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SecureMigrationCandidateFileIo.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateDurabilityTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/CommittedAtomicReplaceTestSupport.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateFileIo.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateFileIo.java new file mode 100644 index 0000000000..879aec1496 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateFileIo.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.io.IOException; +import java.nio.file.Path; + +/** Narrow root-bound publication and durability boundary for migration candidate files. */ +interface MigrationCandidateFileIo { + + /** Consumes the caller-owned bytes synchronously without retaining or modifying them. */ + void publish(Path target, byte[] content) throws IOException; + + /** Confirms prior candidate directory-entry mutations without changing candidate bytes. */ + void confirmDurability(Path target) throws IOException; +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateStore.java index 2461568deb..4ec0c5a58c 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateStore.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateStore.java @@ -13,7 +13,7 @@ import java.nio.file.LinkOption; import java.nio.file.Path; import java.util.Arrays; import java.util.Objects; -import java.util.UUID; +import org.apache.hertzbeat.manager.setup.security.CommittedSetupFileDurabilityException; import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; /** Root-bound file store for generation-scoped, owner-only migration candidates. */ @@ -27,34 +27,46 @@ final class MigrationCandidateStore { private final Path candidateRoot; private final ManagedApplicationConfigStore applicationStore; private final ManagedSecretStore secretStore; + private final MigrationCandidateFileIo fileIo; private final ApplicationConfigDocumentCodec applicationCodec = new ApplicationConfigDocumentCodec(); private final SecretConfigDocumentCodec secretCodec = new SecretConfigDocumentCodec(); private final MigrationCandidateManifestCodec manifestCodec = new MigrationCandidateManifestCodec(); MigrationCandidateStore(Path installationRoot) { + this(installationRoot, null); + } + + MigrationCandidateStore(Path installationRoot, MigrationCandidateFileIo fileIo) { root = prepareRoot(installationRoot); candidateRoot = root.resolve("data/config/migration-candidates"); applicationStore = new FileManagedApplicationConfigStore(root); secretStore = new FileManagedSecretStore(root); + this.fileIo = fileIo == null ? new SecureMigrationCandidateFileIo(root) : fileIo; } ManagedMigrationConfigurationTransaction.StageOutcome stage( ManagedMigrationConfigurationTransaction.CandidateRef reference, String baseGeneration, String targetIdentityHash, ManagedConfigurationBundle bundle) throws IOException { + boolean confirmExisting = false; try (MigrationCandidateMaterial existing = read(reference)) { if (existing.inspection().state() == ManagedMigrationConfigurationTransaction.CandidateState.READY) { boolean same = existing.manifest().filter(manifest -> manifest.baseGeneration().equals(baseGeneration) && manifest.targetIdentityHash().equals(targetIdentityHash)).isPresent() && existing.application().filter(bundle.application()::equals).isPresent() && existing.secrets().filter(bundle.secrets()::equals).isPresent(); - return same ? ManagedMigrationConfigurationTransaction.StageOutcome.ALREADY_STAGED - : ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED; + if (!same) { + return ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED; + } + confirmExisting = true; } - if (existing.inspection().state() + if (!confirmExisting && existing.inspection().state() != ManagedMigrationConfigurationTransaction.CandidateState.MISSING) { return ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED; } } + if (confirmExisting) { + return confirmReady(reference); + } ActivePairState activePair = activePairState(baseGeneration); if (activePair != ActivePairState.MATCH) { return activePair == ActivePairState.STALE @@ -67,14 +79,18 @@ final class MigrationCandidateStore { byte[] manifest = manifestCodec.encode(new MigrationCandidateManifest( reference.operationId(), reference.candidateGeneration(), baseGeneration, targetIdentityHash)); try { - publish(paths.application(), application); - publish(paths.secrets(), secrets); - publish(paths.manifest(), manifest); + fileIo.publish(paths.application(), application); + fileIo.publish(paths.secrets(), secrets); + fileIo.publish(paths.manifest(), manifest); + boolean ready; try (MigrationCandidateMaterial staged = read(reference)) { - return staged.inspection().state() == ManagedMigrationConfigurationTransaction.CandidateState.READY - ? ManagedMigrationConfigurationTransaction.StageOutcome.STAGED - : ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED; + ready = staged.inspection().state() + == ManagedMigrationConfigurationTransaction.CandidateState.READY; } + return ready ? confirmReady(reference, ManagedMigrationConfigurationTransaction.StageOutcome.STAGED) + : ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED; + } catch (CommittedSetupFileDurabilityException uncertain) { + return ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED; } finally { clear(application); clear(secrets); @@ -82,6 +98,22 @@ final class MigrationCandidateStore { } } + private ManagedMigrationConfigurationTransaction.StageOutcome confirmReady( + ManagedMigrationConfigurationTransaction.CandidateRef reference) { + return confirmReady(reference, ManagedMigrationConfigurationTransaction.StageOutcome.ALREADY_STAGED); + } + + private ManagedMigrationConfigurationTransaction.StageOutcome confirmReady( + ManagedMigrationConfigurationTransaction.CandidateRef reference, + ManagedMigrationConfigurationTransaction.StageOutcome confirmedOutcome) { + try { + fileIo.confirmDurability(paths(reference).manifest()); + return confirmedOutcome; + } catch (IOException failure) { + return ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED; + } + } + ManagedMigrationConfigurationTransaction.Inspection inspect( ManagedMigrationConfigurationTransaction.CandidateRef reference) { try (MigrationCandidateMaterial material = read(reference)) { @@ -208,18 +240,6 @@ final class MigrationCandidateStore { directory.resolve("manifest")); } - private void publish(Path target, byte[] content) throws IOException { - Path temporary = target.resolveSibling("." + target.getFileName() + "-" + UUID.randomUUID() + ".tmp"); - try { - SecureSetupFile.create(root, temporary, content); - SecureSetupFile.atomicReplace(root, temporary, target); - } finally { - if (Files.exists(temporary, LinkOption.NOFOLLOW_LINKS)) { - SecureSetupFile.deleteOwnerOnlyInsideRoot(root, temporary); - } - } - } - private void delete(Path target) throws IOException { if (Files.exists(target, LinkOption.NOFOLLOW_LINKS)) { SecureSetupFile.deleteOwnerOnlyInsideRoot(root, target); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SecureMigrationCandidateFileIo.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SecureMigrationCandidateFileIo.java new file mode 100644 index 0000000000..a9552c11b1 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/SecureMigrationCandidateFileIo.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.util.Objects; +import java.util.UUID; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; + +/** SecureSetupFile-backed candidate publication bound to one trusted installation root. */ +final class SecureMigrationCandidateFileIo implements MigrationCandidateFileIo { + + private final Path root; + private final ParentDirectorySync parentDirectorySync; + + SecureMigrationCandidateFileIo(Path trustedRoot) { + this(trustedRoot, (root, target) -> SecureSetupFile.forceParentDirectoryIfSupported(root, target)); + } + + SecureMigrationCandidateFileIo(Path trustedRoot, ParentDirectorySync parentDirectorySync) { + root = Objects.requireNonNull(trustedRoot, "trustedRoot"); + this.parentDirectorySync = Objects.requireNonNull(parentDirectorySync, "parentDirectorySync"); + } + + @Override + public void publish(Path target, byte[] content) throws IOException { + Path temporary = target.resolveSibling("." + target.getFileName() + "-" + UUID.randomUUID() + ".tmp"); + try { + SecureSetupFile.create(root, temporary, content); + SecureSetupFile.atomicReplace(root, temporary, target); + } finally { + if (Files.exists(temporary, LinkOption.NOFOLLOW_LINKS)) { + SecureSetupFile.deleteOwnerOnlyInsideRoot(root, temporary); + } + } + } + + @Override + public void confirmDurability(Path target) throws IOException { + Path manifest = target.toAbsolutePath().normalize(); + Path generation = manifest.getParent(); + Path operation = generation == null ? null : generation.getParent(); + Path candidateRoot = root.resolve("data/config/migration-candidates").normalize(); + if (!"manifest".equals(String.valueOf(manifest.getFileName())) + || operation == null || !candidateRoot.equals(operation.getParent())) { + throw new IOException("Managed migration candidate hierarchy is invalid"); + } + parentDirectorySync.force(root, manifest); + parentDirectorySync.force(root, generation); + parentDirectorySync.force(root, operation); + parentDirectorySync.force(root, candidateRoot); + } + + @FunctionalInterface + interface ParentDirectorySync { + void force(Path trustedRoot, Path target) throws IOException; + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateDurabilityTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateDurabilityTest.java new file mode 100644 index 0000000000..b50390badb --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateDurabilityTest.java @@ -0,0 +1,316 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MailSecurity; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.security.CommittedAtomicReplaceTestSupport; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class MigrationCandidateDurabilityTest { + + private static final String OPERATION = "migration-operation"; + private static final String CANDIDATE = "candidate-generation"; + private static final String IDENTITY = "a".repeat(64); + + @TempDir + private Path installationRoot; + + @Test + void cleanFirstStageConfirmsCandidateHierarchyBeforeReportingStaged() throws Exception { + try (ManagedConfigurationBundle active = bundle("active")) { + new ManagedConfigurationTransaction(installationRoot).apply(active); + } + String baseGeneration = new FileManagedApplicationConfigStore(installationRoot) + .readActive().generation().orElseThrow(); + RecordingCandidateFileIo fileIo = new RecordingCandidateFileIo(installationRoot); + MigrationCandidateStore store = new MigrationCandidateStore(installationRoot, fileIo); + ManagedMigrationConfigurationTransaction.CandidateRef reference = + new ManagedMigrationConfigurationTransaction.CandidateRef(OPERATION, CANDIDATE); + + try (ManagedConfigurationBundle candidate = bundle("candidate")) { + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.STAGED, + store.stage(reference, baseGeneration, IDENTITY, candidate)); + } + + assertEquals(1, fileIo.confirmations()); + } + + @Test + void cleanFirstStageConfirmationFailureCannotReportStaged() throws Exception { + try (ManagedConfigurationBundle active = bundle("active")) { + new ManagedConfigurationTransaction(installationRoot).apply(active); + } + String baseGeneration = new FileManagedApplicationConfigStore(installationRoot) + .readActive().generation().orElseThrow(); + RecordingCandidateFileIo fileIo = new RecordingCandidateFileIo(installationRoot, 1); + MigrationCandidateStore store = new MigrationCandidateStore(installationRoot, fileIo); + ManagedMigrationConfigurationTransaction.CandidateRef reference = + new ManagedMigrationConfigurationTransaction.CandidateRef(OPERATION, CANDIDATE); + + try (ManagedConfigurationBundle candidate = bundle("candidate")) { + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED, + store.stage(reference, baseGeneration, IDENTITY, candidate)); + assertEquals(ManagedMigrationConfigurationTransaction.CandidateState.READY, + store.inspect(reference).state()); + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.ALREADY_STAGED, + store.stage(reference, baseGeneration, IDENTITY, candidate)); + } + + assertEquals(2, fileIo.confirmations()); + } + + @Test + void secureConfirmationForcesEveryNewCandidateAncestorThroughTheConfigBoundary() throws Exception { + Path root = SecureSetupFile.prepareTrustedRoot(installationRoot); + Path candidateRoot = root.resolve("data/config/migration-candidates"); + Path operation = candidateRoot.resolve(OPERATION); + Path generation = operation.resolve(CANDIDATE); + Path manifest = generation.resolve("manifest"); + List confirmedTargets = new ArrayList<>(); + SecureMigrationCandidateFileIo fileIo = new SecureMigrationCandidateFileIo( + root, (trustedRoot, target) -> { + assertEquals(root, trustedRoot); + confirmedTargets.add(target); + }); + + fileIo.confirmDurability(manifest); + + assertEquals(List.of(manifest, generation, operation, candidateRoot), confirmedTargets); + } + + @Test + void failureAtEveryCandidateAncestorNeverReportsStaged() throws Exception { + for (int failureIndex = 0; failureIndex < 4; failureIndex++) { + Path scenarioRoot = installationRoot.resolve("ancestor-failure-" + failureIndex); + try (ManagedConfigurationBundle active = bundle("active-" + failureIndex)) { + new ManagedConfigurationTransaction(scenarioRoot).apply(active); + } + String baseGeneration = new FileManagedApplicationConfigStore(scenarioRoot) + .readActive().generation().orElseThrow(); + Path root = SecureSetupFile.prepareTrustedRoot(scenarioRoot); + AtomicInteger confirmations = new AtomicInteger(); + int failedAncestor = failureIndex; + SecureMigrationCandidateFileIo fileIo = new SecureMigrationCandidateFileIo( + root, (trustedRoot, target) -> { + if (confirmations.getAndIncrement() == failedAncestor) { + throw new IOException("simulated ancestor confirmation failure"); + } + SecureSetupFile.forceParentDirectoryIfSupported(trustedRoot, target); + }); + MigrationCandidateStore store = new MigrationCandidateStore(scenarioRoot, fileIo); + String operationId = OPERATION + "-" + failureIndex; + String candidateGeneration = CANDIDATE + "-" + failureIndex; + ManagedMigrationConfigurationTransaction.CandidateRef reference = + new ManagedMigrationConfigurationTransaction.CandidateRef( + operationId, candidateGeneration); + + try (ManagedConfigurationBundle candidate = bundle("candidate-" + failureIndex)) { + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED, + store.stage(reference, baseGeneration, IDENTITY, candidate)); + } + + assertEquals(ManagedMigrationConfigurationTransaction.CandidateState.READY, + store.inspect(reference).state()); + assertEquals(failureIndex + 1, confirmations.get()); + } + } + + @Test + void exactReadyRetryConfirmsCandidateDirectoryBeforeReportingAlreadyStaged() throws Exception { + try (ManagedConfigurationBundle active = bundle("active")) { + assertEquals(ManagedConfigurationTransaction.Outcome.APPLIED, + new ManagedConfigurationTransaction(installationRoot).apply(active)); + } + String baseGeneration = new FileManagedApplicationConfigStore(installationRoot) + .readActive().generation().orElseThrow(); + Path candidateDirectory = installationRoot.resolve("data/config/migration-candidates") + .resolve(OPERATION).resolve(CANDIDATE); + FailingCandidateFileIo fileIo = new FailingCandidateFileIo(installationRoot); + MigrationCandidateStore store = new MigrationCandidateStore(installationRoot, fileIo); + ManagedMigrationConfigurationTransaction.CandidateRef reference = + new ManagedMigrationConfigurationTransaction.CandidateRef(OPERATION, CANDIDATE); + Map managedBefore = nonMigrationConfigBytes(); + + try (ManagedConfigurationBundle candidate = bundle("candidate")) { + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED, + store.stage(reference, baseGeneration, IDENTITY, candidate)); + assertEquals(ManagedMigrationConfigurationTransaction.CandidateState.READY, + store.inspect(reference).state()); + Map candidateBeforeRetry = fileBytes(candidateDirectory); + Map candidateTimes = fileTimes(candidateDirectory); + + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED, + store.stage(reference, baseGeneration, IDENTITY, candidate)); + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.RECOVERY_REQUIRED, + store.stage(reference, baseGeneration, IDENTITY, candidate)); + assertEquals(ManagedMigrationConfigurationTransaction.StageOutcome.ALREADY_STAGED, + store.stage(reference, baseGeneration, IDENTITY, candidate)); + + assertBytesEqual(candidateBeforeRetry, fileBytes(candidateDirectory)); + assertEquals(candidateTimes, fileTimes(candidateDirectory)); + } + + assertEquals(1, fileIo.committedManifestFailures()); + assertEquals(3, fileIo.confirmations()); + assertBytesEqual(managedBefore, nonMigrationConfigBytes()); + } + + private Map nonMigrationConfigBytes() throws IOException { + Path config = installationRoot.resolve("data/config"); + Path candidates = config.resolve("migration-candidates"); + Map bytes = new LinkedHashMap<>(); + try (Stream files = Files.walk(config)) { + for (Path path : files.filter(candidate -> Files.isRegularFile(candidate, LinkOption.NOFOLLOW_LINKS)) + .filter(candidate -> !candidate.startsWith(candidates)).sorted().toList()) { + bytes.put(config.relativize(path), Files.readAllBytes(path)); + } + } + return bytes; + } + + private static Map fileBytes(Path directory) throws IOException { + Map bytes = new LinkedHashMap<>(); + try (Stream files = Files.list(directory)) { + for (Path path : files.sorted().toList()) { + bytes.put(path.getFileName(), Files.readAllBytes(path)); + } + } + return bytes; + } + + private static Map fileTimes(Path directory) throws IOException { + Map times = new LinkedHashMap<>(); + try (Stream files = Files.list(directory)) { + for (Path path : files.sorted().toList()) { + times.put(path.getFileName(), Files.getLastModifiedTime(path)); + } + } + return times; + } + + private static void assertBytesEqual(Map expected, Map actual) { + assertEquals(expected.keySet(), actual.keySet()); + expected.forEach((path, content) -> assertArrayEquals(content, actual.get(path), path.toString())); + } + + private static ManagedConfigurationBundle bundle(String suffix) { + ManagedOptionalConfiguration.MailSettings mail = new ManagedOptionalConfiguration.MailSettings( + "smtp.example", 587, MailSecurity.STARTTLS, Optional.of("mailer"), "alerts@example.org"); + ManagedApplicationConfig application = new ManagedApplicationConfig( + new MetadataDatabaseSettings(MetadataDatabaseKind.H2, + "jdbc:h2:file:./data/" + suffix, "hertzbeat"), + new GreptimeSettings(new GreptimeEndpoints("greptime:4001", "http://greptime:4000"), + "public", Optional.of("telemetry-user")), + new ManagedOptionalConfiguration(Optional.empty(), Optional.empty(), Optional.of(mail))); + return new ManagedConfigurationBundle(application, + new ManagedSecrets(SecretValue.of("database-" + suffix), + Optional.of(SecretValue.of("telemetry-" + suffix)), + Optional.of(SecretValue.of("mail-" + suffix)))); + } + + private static final class FailingCandidateFileIo implements MigrationCandidateFileIo { + + private final Path root; + private final AtomicBoolean failManifest = new AtomicBoolean(true); + private final AtomicInteger committedManifestFailures = new AtomicInteger(); + private final AtomicInteger confirmations = new AtomicInteger(); + + private FailingCandidateFileIo(Path root) throws IOException { + this.root = SecureSetupFile.prepareTrustedRoot(root); + } + + @Override + public void publish(Path target, byte[] content) throws IOException { + Path temporary = target.resolveSibling("." + target.getFileName() + "-" + UUID.randomUUID() + ".tmp"); + try { + SecureSetupFile.create(root, temporary, content); + if (target.getFileName().toString().equals("manifest") && failManifest.compareAndSet(true, false)) { + committedManifestFailures.incrementAndGet(); + CommittedAtomicReplaceTestSupport.replaceThenFailParentForce( + root, temporary, target); + } else { + SecureSetupFile.atomicReplace(root, temporary, target); + } + } finally { + if (Files.exists(temporary, LinkOption.NOFOLLOW_LINKS)) { + SecureSetupFile.deleteOwnerOnlyInsideRoot(root, temporary); + } + } + } + + @Override + public void confirmDurability(Path target) throws IOException { + if (confirmations.incrementAndGet() <= 2) { + throw new IOException("simulated candidate-directory confirmation failure"); + } + SecureSetupFile.forceParentDirectoryIfSupported(root, target); + } + + private int committedManifestFailures() { + return committedManifestFailures.get(); + } + + private int confirmations() { + return confirmations.get(); + } + } + + private static final class RecordingCandidateFileIo implements MigrationCandidateFileIo { + + private final MigrationCandidateFileIo delegate; + private final AtomicInteger confirmations = new AtomicInteger(); + private final AtomicInteger failuresRemaining; + + private RecordingCandidateFileIo(Path root) throws IOException { + this(root, 0); + } + + private RecordingCandidateFileIo(Path root, int failures) throws IOException { + delegate = new SecureMigrationCandidateFileIo(SecureSetupFile.prepareTrustedRoot(root)); + failuresRemaining = new AtomicInteger(failures); + } + + @Override + public void publish(Path target, byte[] content) throws IOException { + delegate.publish(target, content); + } + + @Override + public void confirmDurability(Path target) throws IOException { + confirmations.incrementAndGet(); + if (failuresRemaining.getAndUpdate(remaining -> Math.max(0, remaining - 1)) > 0) { + throw new IOException("simulated clean-stage confirmation failure"); + } + delegate.confirmDurability(target); + } + + private int confirmations() { + return confirmations.get(); + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/CommittedAtomicReplaceTestSupport.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/CommittedAtomicReplaceTestSupport.java new file mode 100644 index 0000000000..7968030041 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/security/CommittedAtomicReplaceTestSupport.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.security; + +import java.io.IOException; +import java.nio.file.Path; + +/** Test-only proof that the atomic replacement committed before parent-directory force failed. */ +public final class CommittedAtomicReplaceTestSupport { + + private CommittedAtomicReplaceTestSupport() { + } + + public static void replaceThenFailParentForce(Path root, Path source, Path target) throws IOException { + SecureSetupFile.atomicReplace(root, source, target, ignored -> { + throw new IOException("simulated parent-directory force failure"); + }); + } +} From 7114c0e71bc1edd81c8f546c27da747efe05f02f Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 14:29:03 +0800 Subject: [PATCH 54/71] Model metadata preparation recovery --- .../setup/api/MigrationContractValidator.java | 44 +++- .../MigrationOperationTransitionPolicy.java | 9 +- .../workflow/MigrationRestartClassifier.java | 6 + ...igrationOperationTransitionPolicyTest.java | 2 +- .../MigrationPendingStateContractTest.java | 212 ++++++++++++++++++ 5 files changed, 259 insertions(+), 14 deletions(-) create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationPendingStateContractTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/MigrationContractValidator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/MigrationContractValidator.java index a5c4e4b93f..343325528f 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/MigrationContractValidator.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/MigrationContractValidator.java @@ -39,7 +39,9 @@ final class MigrationContractValidator { SetupErrorCode.MIGRATION_COPY_FAILED, SetupErrorCode.MIGRATION_VERIFICATION_FAILED, SetupErrorCode.MIGRATION_ACTIVATION_FAILED, - SetupErrorCode.RESTART_FAILED); + SetupErrorCode.RESTART_FAILED, + SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED, + SetupErrorCode.OPERATION_CONFLICT); private static final Set CAPABILITY_BLOCKERS = Set.of( SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED, SetupErrorCode.MIGRATION_MULTI_NODE_UNSUPPORTED, @@ -107,20 +109,22 @@ final class MigrationContractValidator { || pollAfterMillis < 0) { invalid("Migration projection is incomplete or out of range"); } - validateTimes(state, createdAt, startedAt, completedAt); + validateTimes(state, errorCode, createdAt, startedAt, completedAt); validateState(state, stage, progress, verification, errorCode, pollAfterMillis); validateOutcome(state, errorCode, activationAvailable, restartRequired, externalApplyRequired); } private static void validateTimes( - MigrationOperationState state, Instant createdAt, Instant startedAt, Instant completedAt) { - boolean pending = state == MigrationOperationState.PENDING; + MigrationOperationState state, SetupErrorCode errorCode, + Instant createdAt, Instant startedAt, Instant completedAt) { + boolean notStarted = state == MigrationOperationState.PENDING + || state == MigrationOperationState.FAILED && preparationFailure(errorCode); boolean terminal = terminal(state); - if (pending != (startedAt == null) || terminal != (completedAt != null)) { + if (notStarted != (startedAt == null) || terminal != (completedAt != null)) { invalid("Migration timestamps do not match lifecycle state"); } if (startedAt != null && startedAt.isBefore(createdAt) - || completedAt != null && completedAt.isBefore(startedAt)) { + || completedAt != null && completedAt.isBefore(startedAt == null ? createdAt : startedAt)) { invalid("Migration timestamps are out of order"); } } @@ -130,7 +134,9 @@ final class MigrationContractValidator { VerificationState verification, SetupErrorCode errorCode, long pollAfterMillis) { boolean valid = switch (state) { case PENDING -> stage == MigrationStage.QUEUED && progress == 0 - && verification == VerificationState.PENDING && pollAfterMillis > 0; + && verification == VerificationState.PENDING + && (errorCode == null && pollAfterMillis > 0 + || errorCode == SetupErrorCode.CONFIG_RECOVERY_REQUIRED && pollAfterMillis == 0); case RUNNING -> running(stage, progress, verification) && pollAfterMillis > 0; case READY_TO_ACTIVATE -> stage == MigrationStage.READY_TO_ACTIVATE && progress == 100 && verification == VerificationState.SUCCEEDED && pollAfterMillis == 0; @@ -140,10 +146,11 @@ final class MigrationContractValidator { && verification == VerificationState.SUCCEEDED && pollAfterMillis > 0; case SUCCEEDED -> stage == MigrationStage.COMPLETED && progress == 100 && verification == VerificationState.SUCCEEDED && pollAfterMillis == 0; - case FAILED -> stage == MigrationStage.FAILED && failureMatches(errorCode, verification, progress) + case FAILED -> stage == MigrationStage.FAILED + && failureMatches(state, errorCode, verification, progress) && pollAfterMillis == 0; case ROLLED_BACK -> stage == MigrationStage.ROLLED_BACK - && failureMatches(errorCode, verification, progress) + && failureMatches(state, errorCode, verification, progress) && pollAfterMillis == 0; }; if (!valid) { @@ -152,7 +159,8 @@ final class MigrationContractValidator { } private static boolean failureMatches( - SetupErrorCode errorCode, VerificationState verification, int progress) { + MigrationOperationState state, SetupErrorCode errorCode, + VerificationState verification, int progress) { if (errorCode == null) { return false; } @@ -161,6 +169,9 @@ final class MigrationContractValidator { case MIGRATION_VERIFICATION_FAILED -> verification == VerificationState.FAILED && progress == 100; case MIGRATION_ACTIVATION_FAILED, RESTART_FAILED -> verification == VerificationState.SUCCEEDED && progress == 100; + case MIGRATION_SOURCE_UNSUPPORTED, OPERATION_CONFLICT -> + state == MigrationOperationState.FAILED + && verification == VerificationState.PENDING && progress == 0; default -> false; }; } @@ -179,9 +190,13 @@ final class MigrationContractValidator { private static void validateOutcome( MigrationOperationState state, SetupErrorCode errorCode, boolean activationAvailable, boolean restartRequired, boolean externalApplyRequired) { - boolean failure = state == MigrationOperationState.FAILED || state == MigrationOperationState.ROLLED_BACK; boolean activatable = state == MigrationOperationState.READY_TO_ACTIVATE; - if (failure != (errorCode != null) || errorCode != null && !OPERATION_ERRORS.contains(errorCode) + boolean validError = switch (state) { + case PENDING -> errorCode == null || errorCode == SetupErrorCode.CONFIG_RECOVERY_REQUIRED; + case FAILED, ROLLED_BACK -> errorCode != null && OPERATION_ERRORS.contains(errorCode); + default -> errorCode == null; + }; + if (!validError || activationAvailable != activatable || restartRequired != (state == MigrationOperationState.AWAITING_RESTART) || externalApplyRequired != (state == MigrationOperationState.AWAITING_EXTERNAL_APPLY)) { @@ -189,6 +204,11 @@ final class MigrationContractValidator { } } + private static boolean preparationFailure(SetupErrorCode errorCode) { + return errorCode == SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED + || errorCode == SetupErrorCode.OPERATION_CONFLICT; + } + private static boolean terminal(MigrationOperationState state) { return state == MigrationOperationState.SUCCEEDED || state == MigrationOperationState.FAILED diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicy.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicy.java index 5242ea22e7..f079dfe97d 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicy.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicy.java @@ -44,7 +44,14 @@ final class MigrationOperationTransitionPolicy { private boolean pendingExit(MigrationOperationSnapshot next) { return runningAt(next, MigrationStage.COPYING, VerificationState.PENDING) - || failedWith(next, SetupErrorCode.MIGRATION_COPY_FAILED); + || blockedPending(next) + || failedWith(next, SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED) + || failedWith(next, SetupErrorCode.OPERATION_CONFLICT); + } + + private boolean blockedPending(MigrationOperationSnapshot next) { + return next.state() == MigrationOperationState.PENDING + && next.errorCode() == SetupErrorCode.CONFIG_RECOVERY_REQUIRED; } private boolean runningExit(MigrationOperationSnapshot current, MigrationOperationSnapshot next) { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationRestartClassifier.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationRestartClassifier.java index 36503d8ef9..9421de338a 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationRestartClassifier.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationRestartClassifier.java @@ -8,8 +8,10 @@ package org.apache.hertzbeat.manager.setup.workflow; import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; /** * Classifies durable migration state after a restart without performing recovery I/O. @@ -20,6 +22,10 @@ final class MigrationRestartClassifier { Plan classify(MigrationOperationSnapshot snapshot, CandidateEvidence evidence) { Objects.requireNonNull(snapshot, "snapshot"); Objects.requireNonNull(evidence, "evidence"); + if (snapshot.state() == MigrationOperationState.PENDING + && snapshot.errorCode() == SetupErrorCode.CONFIG_RECOVERY_REQUIRED) { + return Plan.RECOVERY_REQUIRED; + } if (evidence == CandidateEvidence.INCONSISTENT || evidence == CandidateEvidence.RECOVERY_REQUIRED) { return Plan.RECOVERY_REQUIRED; diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicyTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicyTest.java index e761745c82..b28099c840 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicyTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationTransitionPolicyTest.java @@ -129,7 +129,7 @@ class MigrationOperationTransitionPolicyTest { MigrationOperationSnapshot rolledBack = rolledBack(MigrationRollbackOrigin.RESTART_FAILURE, SetupErrorCode.RESTART_FAILED); - assertAllowed(pending, failed(SetupErrorCode.MIGRATION_COPY_FAILED)); + assertRejected(pending, failed(SetupErrorCode.MIGRATION_COPY_FAILED)); assertAllowed(copying, failed(SetupErrorCode.MIGRATION_COPY_FAILED)); assertAllowed(verifying, failed(SetupErrorCode.MIGRATION_VERIFICATION_FAILED)); assertAllowed(activating, failed(SetupErrorCode.MIGRATION_ACTIVATION_FAILED)); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationPendingStateContractTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationPendingStateContractTest.java new file mode 100644 index 0000000000..ac019604f5 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationPendingStateContractTest.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.nio.file.Path; +import java.time.Instant; +import java.util.List; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.workflow.MigrationRestartClassifier.CandidateEvidence; +import org.apache.hertzbeat.manager.setup.workflow.MigrationRestartClassifier.Plan; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class MigrationPendingStateContractTest { + + private static final Instant CREATED = Instant.parse("2026-08-10T01:00:00Z"); + private static final String IDENTITY = "a".repeat(64); + private static final String GENERATION = "candidate-generation"; + + @TempDir + private Path root; + + private final MigrationOperationTransitionPolicy transitions = new MigrationOperationTransitionPolicy(); + private final MigrationRestartClassifier restart = new MigrationRestartClassifier(); + + @Test + void distinguishesCleanAndRecoveryBlockedPendingWithoutInventingWork() { + MigrationOperationSnapshot clean = cleanPending("operation-clean", ApplyMode.MANAGED_WRITE); + MigrationOperationSnapshot blocked = blockedPending("operation-blocked", ApplyMode.MANAGED_WRITE); + + assertThat(clean.errorCode()).isNull(); + assertThat(clean.nextPollAfterMillis()).isPositive(); + assertThat(blocked.errorCode()).isEqualTo(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + assertThat(blocked.nextPollAfterMillis()).isZero(); + assertThat(blocked.startedAt()).isNull(); + assertThat(blocked.completedAt()).isNull(); + assertThat(blocked.progressPercent()).isZero(); + assertThat(blocked.verificationState()).isEqualTo(VerificationState.PENDING); + assertThatThrownBy(() -> snapshot("operation-a", ApplyMode.MANAGED_WRITE, + MigrationOperationState.PENDING, MigrationStage.QUEUED, 0, null, null, + VerificationState.PENDING, SetupErrorCode.CONFIG_RECOVERY_REQUIRED, 1000)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> snapshot("operation-a", ApplyMode.MANAGED_WRITE, + MigrationOperationState.PENDING, MigrationStage.QUEUED, 0, CREATED, null, + VerificationState.PENDING, SetupErrorCode.CONFIG_RECOVERY_REQUIRED, 0)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> snapshot("operation-a", ApplyMode.MANAGED_WRITE, + MigrationOperationState.PENDING, MigrationStage.QUEUED, 0, null, CREATED.plusSeconds(1), + VerificationState.PENDING, SetupErrorCode.CONFIG_RECOVERY_REQUIRED, 0)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> snapshot("operation-a", ApplyMode.MANAGED_WRITE, + MigrationOperationState.PENDING, MigrationStage.QUEUED, 0, null, null, + VerificationState.PENDING, SetupErrorCode.CONFIG_WRITE_FAILED, 0)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void onlyProvenSourceAndStalePreparationOutcomesMayTerminateBeforeStart() { + MigrationOperationSnapshot source = preparationFailed( + "operation-source", SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED); + MigrationOperationSnapshot stale = preparationFailed( + "operation-stale", SetupErrorCode.OPERATION_CONFLICT); + + assertThat(source.startedAt()).isNull(); + assertThat(source.completedAt()).isEqualTo(CREATED.plusSeconds(1)); + assertThat(stale.startedAt()).isNull(); + assertThat(stale.completedAt()).isEqualTo(CREATED.plusSeconds(1)); + assertThatThrownBy(() -> failedBeforeStart(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, 0, + VerificationState.PENDING, null)).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> failedBeforeStart(SetupErrorCode.MIGRATION_COPY_FAILED, 0, + VerificationState.PENDING, null)).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> failedBeforeStart(SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED, 1, + VerificationState.PENDING, null)).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> failedBeforeStart(SetupErrorCode.OPERATION_CONFLICT, 0, + VerificationState.SUCCEEDED, null)).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> failedBeforeStart(SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED, 0, + VerificationState.PENDING, CREATED)).isInstanceOf(IllegalArgumentException.class); + } + + @Test + void pendingTransitionGraphKeepsRecoveryOwnershipAndRejectsFalseCopyFailure() { + MigrationOperationSnapshot clean = cleanPending("operation-a", ApplyMode.MANAGED_WRITE); + MigrationOperationSnapshot blocked = blockedPending("operation-a", ApplyMode.MANAGED_WRITE); + MigrationOperationSnapshot running = running("operation-a", ApplyMode.MANAGED_WRITE); + + assertAllowed(clean, blocked); + assertAllowed(blocked, blocked); + assertAllowed(blocked, running); + assertAllowed(clean, preparationFailed("operation-a", SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED)); + assertAllowed(blocked, preparationFailed("operation-a", SetupErrorCode.OPERATION_CONFLICT)); + assertRejected(clean, copyFailed("operation-a")); + assertRejected(blocked, copyFailed("operation-a")); + assertAllowed(running, copyFailed("operation-a")); + } + + @Test + void blockedPendingAlwaysRequiresRecoveryBeforeCredentialPlans() { + for (ApplyMode mode : ApplyMode.values()) { + MigrationOperationSnapshot blocked = blockedPending("operation-a", mode); + for (CandidateEvidence evidence : CandidateEvidence.values()) { + assertThat(restart.classify(blocked, evidence)).isEqualTo(Plan.RECOVERY_REQUIRED); + } + } + } + + @Test + void terminalPreparationFailuresKeepExistingRestartCleanupRules() { + MigrationOperationSnapshot managed = preparationFailed( + "operation-managed", SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED); + MigrationOperationSnapshot external = failedBeforeStart( + "operation-external", ApplyMode.EXTERNAL_APPLY, SetupErrorCode.OPERATION_CONFLICT, + 0, VerificationState.PENDING, null); + + assertThat(restart.classify(managed, CandidateEvidence.EXACT)) + .isEqualTo(Plan.CLEANUP_TERMINAL_CANDIDATE); + assertThat(restart.classify(managed, CandidateEvidence.MISSING)).isEqualTo(Plan.NONE); + assertThat(restart.classify(external, CandidateEvidence.NOT_APPLICABLE)).isEqualTo(Plan.NONE); + } + + @Test + void codecAndExactStoreRoundTripBlockedAndTerminalPreparationStates() { + MigrationOperationSnapshot blocked = blockedPending("operation-a", ApplyMode.MANAGED_WRITE); + MigrationOperationSnapshot terminal = preparationFailed( + "operation-b", SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED); + MigrationOperationFileCodec codec = new MigrationOperationFileCodec(); + + assertThat(codec.decode(codec.encode(List.of(terminal, blocked)))) + .containsExactly(terminal, blocked); + + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + MigrationOperationSnapshot clean = cleanPending("operation-a", ApplyMode.MANAGED_WRITE); + assertThat(store.createOrConfirm(clean)).isEqualTo(clean); + assertThat(store.compareAndTransitionOrConfirm( + clean.operationId(), MigrationOperationState.PENDING, blocked)).isEqualTo(blocked); + assertThat(store.compareAndTransitionOrConfirm( + blocked.operationId(), MigrationOperationState.PENDING, blocked)).isEqualTo(blocked); + MigrationOperationSnapshot running = running("operation-a", ApplyMode.MANAGED_WRITE); + assertThat(store.compareAndTransitionOrConfirm( + blocked.operationId(), MigrationOperationState.PENDING, running)).isEqualTo(running); + assertThat(store.find(blocked.operationId())).contains(running); + } + + private static MigrationOperationSnapshot cleanPending(String operationId, ApplyMode mode) { + return snapshot(operationId, mode, MigrationOperationState.PENDING, MigrationStage.QUEUED, + 0, null, null, VerificationState.PENDING, null, 1000); + } + + private static MigrationOperationSnapshot blockedPending(String operationId, ApplyMode mode) { + return snapshot(operationId, mode, MigrationOperationState.PENDING, MigrationStage.QUEUED, + 0, null, null, VerificationState.PENDING, SetupErrorCode.CONFIG_RECOVERY_REQUIRED, 0); + } + + private static MigrationOperationSnapshot preparationFailed(String operationId, SetupErrorCode error) { + return failedBeforeStart(operationId, ApplyMode.MANAGED_WRITE, error, 0, + VerificationState.PENDING, null); + } + + private static MigrationOperationSnapshot failedBeforeStart( + SetupErrorCode error, int progress, VerificationState verification, Instant startedAt) { + return failedBeforeStart("operation-a", ApplyMode.MANAGED_WRITE, error, progress, verification, startedAt); + } + + private static MigrationOperationSnapshot failedBeforeStart( + String operationId, ApplyMode mode, SetupErrorCode error, + int progress, VerificationState verification, Instant startedAt) { + return snapshot(operationId, mode, MigrationOperationState.FAILED, MigrationStage.FAILED, + progress, startedAt, CREATED.plusSeconds(1), verification, error, 0); + } + + private static MigrationOperationSnapshot running(String operationId, ApplyMode mode) { + return snapshot(operationId, mode, MigrationOperationState.RUNNING, MigrationStage.COPYING, + 10, CREATED.plusSeconds(1), null, VerificationState.PENDING, null, 1000); + } + + private static MigrationOperationSnapshot copyFailed(String operationId) { + return snapshot(operationId, ApplyMode.MANAGED_WRITE, MigrationOperationState.FAILED, + MigrationStage.FAILED, 10, CREATED.plusSeconds(1), CREATED.plusSeconds(2), + VerificationState.PENDING, SetupErrorCode.MIGRATION_COPY_FAILED, 0); + } + + private static MigrationOperationSnapshot snapshot( + String operationId, ApplyMode mode, MigrationOperationState state, MigrationStage stage, + int progress, Instant startedAt, Instant completedAt, VerificationState verification, + SetupErrorCode error, long pollMillis) { + return new MigrationOperationSnapshot(operationId, state, MigrationTarget.MYSQL, mode, stage, + progress, CREATED, startedAt, completedAt, verification, error, null, pollMillis, + false, false, false, IDENTITY, mode == ApplyMode.MANAGED_WRITE ? GENERATION : null); + } + + private void assertAllowed(MigrationOperationSnapshot current, MigrationOperationSnapshot next) { + assertThatCode(() -> transitions.requireAllowed(current, next)).doesNotThrowAnyException(); + } + + private void assertRejected(MigrationOperationSnapshot current, MigrationOperationSnapshot next) { + assertThatThrownBy(() -> transitions.requireAllowed(current, next)) + .isInstanceOf(MigrationOperationStoreException.class); + } +} From c83565b818a2e12ffd0d084d6cd21951deb55058 Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 14:54:28 +0800 Subject: [PATCH 55/71] Prepare metadata cutovers durably --- .../setup/workflow/DurableCutoverDraft.java | 52 ++ .../workflow/DurableCutoverPreparation.java | 186 ++++++++ .../DurableCutoverPreparationException.java | 27 ++ .../workflow/DurableCutoverSnapshots.java | 86 ++++ .../workflow/FileMigrationOperationStore.java | 15 +- .../workflow/RetainedCutoverCoordinator.java | 2 + .../workflow/RetainedCutoverOutcome.java | 1 + .../workflow/RetainedCutoverPreparation.java | 18 +- .../setup/workflow/RetainedCutoverSteps.java | 4 +- .../DurableCutoverPreparationTest.java | 445 ++++++++++++++++++ .../FileMigrationOperationStoreExactTest.java | 11 +- .../RetainedCutoverPreparationTest.java | 35 +- 12 files changed, 858 insertions(+), 24 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverDraft.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverPreparation.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverPreparationException.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverSnapshots.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverPreparationTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverDraft.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverDraft.java new file mode 100644 index 0000000000..559b3f09c3 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverDraft.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.time.Instant; +import java.util.Objects; +import java.util.regex.Pattern; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.OperationIdValidator; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; + +/** Immutable, secret-free identity and timestamps for one durable cutover preparation. */ +record DurableCutoverDraft( + String operationId, + MigrationTarget target, + ApplyMode applyMode, + Instant createdAt, + Instant startedAt, + String candidateGeneration) { + + private static final Pattern GENERATION = Pattern.compile("[A-Za-z0-9][A-Za-z0-9-]{0,63}"); + + DurableCutoverDraft { + if (!OperationIdValidator.isSafe(operationId)) { + throw new IllegalArgumentException("Unsafe migration operation identifier"); + } + Objects.requireNonNull(target, "target"); + Objects.requireNonNull(applyMode, "applyMode"); + Objects.requireNonNull(createdAt, "createdAt"); + Objects.requireNonNull(startedAt, "startedAt"); + if (startedAt.isBefore(createdAt)) { + throw new IllegalArgumentException("Migration start cannot precede creation"); + } + boolean managedGeneration = candidateGeneration != null + && GENERATION.matcher(candidateGeneration).matches(); + if (applyMode == ApplyMode.MANAGED_WRITE && !managedGeneration + || applyMode == ApplyMode.EXTERNAL_APPLY && candidateGeneration != null) { + throw new IllegalArgumentException("Migration candidate does not match apply mode"); + } + } + + @Override + public String toString() { + return "DurableCutoverDraft[operationId=" + operationId + ", target=" + target + + ", applyMode=" + applyMode + "]"; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverPreparation.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverPreparation.java new file mode 100644 index 0000000000..b8d2144bef --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverPreparation.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.IOException; +import java.util.Objects; +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.CandidateRef; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.MetadataTargetStageResult; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.apache.hertzbeat.manager.setup.workflow.FileMigrationOperationStore.ExactTransitionDisposition; + +/** Implements the durable journal-and-candidate boundary before target provisioning. */ +final class DurableCutoverPreparation implements RetainedCutoverPreparation { + + private final DurableCutoverDraft draft; + private final FileMigrationOperationStore store; + private final ManagedMigrationConfigurationTransaction configuration; + + DurableCutoverPreparation( + DurableCutoverDraft draft, + FileMigrationOperationStore store, + ManagedMigrationConfigurationTransaction configuration) { + this.draft = Objects.requireNonNull(draft, "draft"); + this.store = Objects.requireNonNull(store, "store"); + this.configuration = Objects.requireNonNull(configuration, "configuration"); + } + + @Override + public void prepare( + RetainedCutoverPreparationContext context, + MetadataDatabaseSettings target, + SecretValue borrowedPassword) { + try { + prepareDurably(context, target, borrowedPassword); + } catch (MigrationOperationStoreException failure) { + throw failure(failure.errorCode()); + } + } + + private void prepareDurably( + RetainedCutoverPreparationContext context, + MetadataDatabaseSettings target, + SecretValue borrowedPassword) { + requireExactRequest(context, target, borrowedPassword); + DurableCutoverSnapshots snapshots = new DurableCutoverSnapshots( + draft, context.targetIdentityHash()); + MigrationOperationSnapshot clean = snapshots.cleanPending(); + MigrationOperationSnapshot blocked = snapshots.blockedPending(); + MigrationOperationSnapshot running = snapshots.running(); + Optional current = store.find(draft.operationId()); + if (current.filter(snapshots::compatibleRunning).isPresent()) { + throw stopAfterConfirm(current.orElseThrow()); + } + if (current.isPresent() && !current.get().equals(clean) && !current.get().equals(blocked)) { + if (!snapshots.sameIdentity(current.get())) { + throw new MigrationOperationStoreException(SetupErrorCode.OPERATION_CONFLICT); + } + throw stopAfterConfirm(current.get()); + } + confirmPending(current, clean, blocked); + if (draft.applyMode() == ApplyMode.MANAGED_WRITE) { + prepareManaged(context, target, borrowedPassword, snapshots, blocked, running); + } else { + transitionToRunning(running); + } + } + + private void confirmPending( + Optional current, + MigrationOperationSnapshot clean, + MigrationOperationSnapshot blocked) { + if (current.filter(blocked::equals).isPresent()) { + store.compareAndTransitionOrConfirm( + draft.operationId(), MigrationOperationState.PENDING, blocked); + return; + } + store.createOrConfirm(clean); + } + + private void prepareManaged( + RetainedCutoverPreparationContext context, + MetadataDatabaseSettings target, + SecretValue borrowedPassword, + DurableCutoverSnapshots snapshots, + MigrationOperationSnapshot blocked, + MigrationOperationSnapshot running) { + MetadataTargetStageResult result; + try (SecretValue ownedPassword = SecretValue.copyOf(borrowedPassword)) { + result = configuration.stageMetadataTarget( + draft.operationId(), draft.candidateGeneration(), context.targetIdentityHash(), + target, ownedPassword); + } catch (IOException | RuntimeException failure) { + transitionAndFail(blocked, SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + return; + } + switch (result.outcome()) { + case STAGED, ALREADY_STAGED -> { + if (!exactCandidate(result)) { + transitionAndFail(blocked, SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + transitionToRunning(running); + } + case SOURCE_UNSUPPORTED -> transitionAndFail( + snapshots.failed(SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED), + SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED); + case STALE -> transitionAndFail( + snapshots.failed(SetupErrorCode.OPERATION_CONFLICT), + SetupErrorCode.OPERATION_CONFLICT); + case RECOVERY_REQUIRED -> transitionAndFail( + blocked, SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + default -> transitionAndFail(blocked, SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + } + + private boolean exactCandidate(MetadataTargetStageResult result) { + return result.candidate().equals(Optional.of( + new CandidateRef(draft.operationId(), draft.candidateGeneration()))); + } + + private void transitionAndFail(MigrationOperationSnapshot replacement, SetupErrorCode code) { + transition(replacement); + throw failure(code); + } + + private void transitionToRunning(MigrationOperationSnapshot running) { + ExactTransitionDisposition disposition = transition(running); + if (disposition == ExactTransitionDisposition.ALREADY_CONFIRMED) { + throw failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + } + + private ExactTransitionDisposition transition(MigrationOperationSnapshot replacement) { + return store.compareAndTransitionOrConfirmDisposition( + draft.operationId(), MigrationOperationState.PENDING, replacement); + } + + private DurableCutoverPreparationException stopAfterConfirm(MigrationOperationSnapshot current) { + ExactTransitionDisposition disposition = store.compareAndTransitionOrConfirmDisposition( + draft.operationId(), current.state(), current); + if (disposition != ExactTransitionDisposition.ALREADY_CONFIRMED) { + throw new MigrationOperationStoreException(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + if ((current.state() == MigrationOperationState.FAILED + || current.state() == MigrationOperationState.ROLLED_BACK) + && current.errorCode() != null) { + return failure(current.errorCode()); + } + return failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + + private void requireExactRequest( + RetainedCutoverPreparationContext context, + MetadataDatabaseSettings target, + SecretValue borrowedPassword) { + Objects.requireNonNull(context, "context"); + Objects.requireNonNull(target, "target"); + Objects.requireNonNull(borrowedPassword, "borrowedPassword"); + if (!draft.operationId().equals(context.operationId()) + || targetKind() != target.kind()) { + throw new MigrationOperationStoreException(SetupErrorCode.OPERATION_CONFLICT); + } + } + + private MetadataDatabaseKind targetKind() { + return switch (draft.target()) { + case MYSQL -> MetadataDatabaseKind.MYSQL; + case POSTGRESQL -> MetadataDatabaseKind.POSTGRESQL; + }; + } + + private static DurableCutoverPreparationException failure(SetupErrorCode code) { + return new DurableCutoverPreparationException(code); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverPreparationException.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverPreparationException.java new file mode 100644 index 0000000000..0acb682028 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverPreparationException.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Stable, cause-free preparation result that prevents target provisioning. */ +final class DurableCutoverPreparationException extends RuntimeException { + + private final SetupErrorCode errorCode; + + DurableCutoverPreparationException(SetupErrorCode errorCode) { + super("Durable cutover preparation failed: " + + Objects.requireNonNull(errorCode, "errorCode").value()); + this.errorCode = errorCode; + } + + SetupErrorCode errorCode() { + return errorCode; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverSnapshots.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverSnapshots.java new file mode 100644 index 0000000000..f7a59226c9 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverSnapshots.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.time.Instant; +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Builds and compares the exact secret-free journal shapes used during preparation. */ +final class DurableCutoverSnapshots { + + private static final long ACTIVE_POLL_MILLIS = 1000; + + private final DurableCutoverDraft draft; + private final String targetIdentityHash; + + DurableCutoverSnapshots(DurableCutoverDraft draft, String targetIdentityHash) { + this.draft = Objects.requireNonNull(draft, "draft"); + this.targetIdentityHash = Objects.requireNonNull(targetIdentityHash, "targetIdentityHash"); + } + + MigrationOperationSnapshot cleanPending() { + return snapshot(MigrationOperationState.PENDING, MigrationStage.QUEUED, 0, + null, null, null, ACTIVE_POLL_MILLIS); + } + + MigrationOperationSnapshot blockedPending() { + return snapshot(MigrationOperationState.PENDING, MigrationStage.QUEUED, 0, + null, null, SetupErrorCode.CONFIG_RECOVERY_REQUIRED, 0); + } + + MigrationOperationSnapshot running() { + return snapshot(MigrationOperationState.RUNNING, MigrationStage.COPYING, 0, + draft.startedAt(), null, null, ACTIVE_POLL_MILLIS); + } + + MigrationOperationSnapshot failed(SetupErrorCode errorCode) { + return snapshot(MigrationOperationState.FAILED, MigrationStage.FAILED, 0, + null, draft.startedAt(), errorCode, 0); + } + + boolean compatibleRunning(MigrationOperationSnapshot current) { + return current.state() == MigrationOperationState.RUNNING + && current.stage() == MigrationStage.COPYING + && sameIdentity(current) + && current.startedAt().equals(draft.startedAt()) + && current.completedAt() == null + && current.verificationState() == VerificationState.PENDING + && current.errorCode() == null + && current.rollbackOrigin() == null + && !current.activationAvailable() + && !current.restartRequired() + && !current.externalApplyRequired(); + } + + boolean sameIdentity(MigrationOperationSnapshot current) { + return current.target() == draft.target() + && current.applyMode() == draft.applyMode() + && current.createdAt().equals(draft.createdAt()) + && current.targetIdentityHash().equals(targetIdentityHash) + && Objects.equals(current.managedCandidateGeneration(), draft.candidateGeneration()); + } + + private MigrationOperationSnapshot snapshot( + MigrationOperationState state, + MigrationStage stage, + int progress, + Instant startedAt, + Instant completedAt, + SetupErrorCode errorCode, + long pollMillis) { + return new MigrationOperationSnapshot( + draft.operationId(), state, draft.target(), draft.applyMode(), stage, progress, + draft.createdAt(), startedAt, completedAt, VerificationState.PENDING, errorCode, + null, pollMillis, false, false, false, targetIdentityHash, + draft.candidateGeneration()); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java index 171a5d7c0a..10e77c550b 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java @@ -117,6 +117,13 @@ public final class FileMigrationOperationStore implements MigrationOperationStor /** Transitions or confirms one fully equal replacement under the store lock. */ MigrationOperationSnapshot compareAndTransitionOrConfirm( String operationId, MigrationOperationState expectedState, MigrationOperationSnapshot replacement) { + compareAndTransitionOrConfirmDisposition(operationId, expectedState, replacement); + return replacement; + } + + /** Transitions or confirms exact state while reporting which action won under the store lock. */ + ExactTransitionDisposition compareAndTransitionOrConfirmDisposition( + String operationId, MigrationOperationState expectedState, MigrationOperationSnapshot replacement) { requireSafeId(operationId); Objects.requireNonNull(expectedState, "expectedState"); Objects.requireNonNull(replacement, "replacement"); @@ -142,7 +149,7 @@ public final class FileMigrationOperationStore implements MigrationOperationStor throw failure(SetupErrorCode.OPERATION_NOT_FOUND); } - private MigrationOperationSnapshot transitionOrConfirm( + private ExactTransitionDisposition transitionOrConfirm( List snapshots, String operationId, MigrationOperationState expectedState, MigrationOperationSnapshot replacement) { for (int index = 0; index < snapshots.size(); index++) { @@ -150,7 +157,7 @@ public final class FileMigrationOperationStore implements MigrationOperationStor if (current.operationId().equals(operationId)) { if (current.equals(replacement)) { writeAndConfirm(snapshots); - return replacement; + return ExactTransitionDisposition.ALREADY_CONFIRMED; } if (current.state() != expectedState) { throw failure(SetupErrorCode.OPERATION_CONFLICT); @@ -159,7 +166,7 @@ public final class FileMigrationOperationStore implements MigrationOperationStor snapshots.set(index, replacement); trim(snapshots); writeAndConfirm(snapshots); - return replacement; + return ExactTransitionDisposition.TRANSITIONED; } } throw failure(SetupErrorCode.OPERATION_NOT_FOUND); @@ -263,4 +270,6 @@ public final class FileMigrationOperationStore implements MigrationOperationStor interface Publisher { void publish(Path target, byte[] content) throws IOException; } + + enum ExactTransitionDisposition { TRANSITIONED, ALREADY_CONFIRMED } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java index ae9ebac510..2765621f1f 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java @@ -59,6 +59,8 @@ final class RetainedCutoverCoordinator { RetainedCutoverOutcome preparationOutcome = steps.prepare( preparation, new RetainedCutoverPreparationContext(operationId, provisionIdentity), + target, + borrowedPassword, deadline); if (!preparationOutcome.successful()) { return finish(execution, RetainedCutoverRelease.resources( diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverOutcome.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverOutcome.java index d0d34f4b91..37a9aff705 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverOutcome.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverOutcome.java @@ -47,6 +47,7 @@ final class RetainedCutoverOutcome { || failure instanceof TargetJdbcConnectionException || failure instanceof TargetSchemaProvisioningException || failure instanceof MigrationMaintenanceException + || failure instanceof DurableCutoverPreparationException || failure instanceof RetainedCutoverException) { return stable((RuntimeException) failure); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverPreparation.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverPreparation.java index 2b7432430e..bd4cf20f76 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverPreparation.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverPreparation.java @@ -7,18 +7,24 @@ package org.apache.hertzbeat.manager.setup.workflow; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; + /** * Persists the secret-free preparation boundary before target schema provisioning begins. * - *

The callback is synchronous and must not retain its context. A normal return means the - * operation is durably prepared for the exact target identity. Callers must pass this seam - * explicitly; {@link #NO_OP} exists only for isolated tests that do not exercise durable workflow - * state. + *

The callback is synchronous and must not retain its context, target, or borrowed password. A + * normal return means the operation is durably prepared for the exact target identity. Callers + * must pass this seam explicitly; {@link #NO_OP} exists only for isolated tests that do not + * exercise durable workflow state. */ @FunctionalInterface interface RetainedCutoverPreparation { - RetainedCutoverPreparation NO_OP = context -> { }; + RetainedCutoverPreparation NO_OP = (context, target, password) -> { }; - void prepare(RetainedCutoverPreparationContext context); + void prepare( + RetainedCutoverPreparationContext context, + MetadataDatabaseSettings target, + SecretValue borrowedPassword); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverSteps.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverSteps.java index f4faa0193d..da087ae3e6 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverSteps.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverSteps.java @@ -43,10 +43,12 @@ final class RetainedCutoverSteps { RetainedCutoverOutcome prepare( RetainedCutoverPreparation preparation, RetainedCutoverPreparationContext context, + MetadataDatabaseSettings target, + SecretValue borrowedPassword, JdbcMetadataMigrationDeadline deadline) { try { requirePreparationBudget(deadline); - preparation.prepare(context); + preparation.prepare(context, target, borrowedPassword); requirePreparationBudget(deadline); return RetainedCutoverOutcome.success(); } catch (RuntimeException | Error failure) { diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverPreparationTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverPreparationTest.java new file mode 100644 index 0000000000..4b5d7c695f --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverPreparationTest.java @@ -0,0 +1,445 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.file.Path; +import java.time.Instant; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.CandidateRef; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.MetadataTargetStageResult; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.StageOutcome; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; + +@Timeout(15) +class DurableCutoverPreparationTest { + + private static final String OPERATION = "operation-a"; + private static final String IDENTITY = "a".repeat(64); + private static final String GENERATION = "candidate-generation"; + private static final Instant CREATED = Instant.parse("2026-08-10T02:00:00Z"); + private static final Instant STARTED = CREATED.plusSeconds(1); + private static final MetadataDatabaseSettings MYSQL = new MetadataDatabaseSettings( + MetadataDatabaseKind.MYSQL, "jdbc:mysql://db.example/hertzbeat", "migration"); + + @TempDir + private Path root; + + @Test + void managedStageIsDurableBeforeExactRunningAndDoesNotRetainBorrowedSecret() throws Exception { + ManagedMigrationConfigurationTransaction configuration = mock( + ManagedMigrationConfigurationTransaction.class); + when(configuration.stageMetadataTarget(eq(OPERATION), eq(GENERATION), eq(IDENTITY), + same(MYSQL), any(SecretValue.class))) + .thenReturn(staged(StageOutcome.STAGED)); + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + DurableCutoverPreparation preparation = managed(store, configuration); + + try (SecretValue borrowed = password(); SecretValue expected = password()) { + preparation.prepare(context(), MYSQL, borrowed); + + assertThat(borrowed).isEqualTo(expected); + } + assertThat(store.find(OPERATION)).contains(running(ApplyMode.MANAGED_WRITE)); + assertThat(store.find(OPERATION).orElseThrow().progressPercent()).isZero(); + ArgumentCaptor owned = ArgumentCaptor.forClass(SecretValue.class); + verify(configuration).stageMetadataTarget( + eq(OPERATION), eq(GENERATION), eq(IDENTITY), same(MYSQL), owned.capture()); + char[] cleared = owned.getValue().copy(); + try { + assertThat(cleared).containsOnly('\0'); + } finally { + Arrays.fill(cleared, '\0'); + } + } + + @Test + void exactAlreadyStagedRetryConvergesWithoutChangingTheDraft() throws Exception { + ManagedMigrationConfigurationTransaction configuration = mock( + ManagedMigrationConfigurationTransaction.class); + when(configuration.stageMetadataTarget(eq(OPERATION), eq(GENERATION), eq(IDENTITY), + same(MYSQL), any(SecretValue.class))) + .thenReturn(staged(StageOutcome.ALREADY_STAGED)); + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + + try (SecretValue borrowed = password()) { + managed(store, configuration).prepare(context(), MYSQL, borrowed); + } + + assertThat(store.find(OPERATION)).contains(running(ApplyMode.MANAGED_WRITE)); + } + + @Test + void blockedPendingRetainsOwnershipAndCanResumeWithTheSameDraft() throws Exception { + ManagedMigrationConfigurationTransaction configuration = mock( + ManagedMigrationConfigurationTransaction.class); + when(configuration.stageMetadataTarget(eq(OPERATION), eq(GENERATION), eq(IDENTITY), + same(MYSQL), any(SecretValue.class))) + .thenReturn(new MetadataTargetStageResult(StageOutcome.RECOVERY_REQUIRED, Optional.empty())) + .thenReturn(staged(StageOutcome.STAGED)); + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + DurableCutoverPreparation preparation = managed(store, configuration); + + try (SecretValue borrowed = password()) { + assertPreparationError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> preparation.prepare(context(), MYSQL, borrowed)); + assertThat(store.find(OPERATION)).contains(blocked(ApplyMode.MANAGED_WRITE)); + preparation.prepare(context(), MYSQL, borrowed); + } + + assertThat(store.find(OPERATION)).contains(running(ApplyMode.MANAGED_WRITE)); + } + + @Test + void sourceUnsupportedAndStaleBecomeTruthfulPreCopyTerminalRecords() throws Exception { + assertTerminalOutcome(StageOutcome.SOURCE_UNSUPPORTED, + SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED); + assertTerminalOutcome(StageOutcome.STALE, SetupErrorCode.OPERATION_CONFLICT); + } + + @Test + void unknownConfigurationFailureBlocksPendingWithoutLeakingCause() throws Exception { + ManagedMigrationConfigurationTransaction configuration = mock( + ManagedMigrationConfigurationTransaction.class); + when(configuration.stageMetadataTarget(eq(OPERATION), eq(GENERATION), eq(IDENTITY), + same(MYSQL), any(SecretValue.class))) + .thenThrow(new IOException("private candidate path")); + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + DurableCutoverPreparation preparation = managed(store, configuration); + + try (SecretValue borrowed = password(); SecretValue expected = password()) { + assertThatThrownBy(() -> preparation.prepare(context(), MYSQL, borrowed)) + .isInstanceOfSatisfying(DurableCutoverPreparationException.class, failure -> { + assertThat(failure.errorCode()).isEqualTo(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + assertThat(failure).hasNoCause(); + assertThat(failure.getMessage()).doesNotContain("private", "path", "jdbc:"); + }); + assertThat(borrowed).isEqualTo(expected); + } + assertThat(store.find(OPERATION)).contains(blocked(ApplyMode.MANAGED_WRITE)); + } + + @Test + void externalPreparationSkipsManagedCandidateAndTransitionsExactlyToRunning() { + ManagedMigrationConfigurationTransaction configuration = mock( + ManagedMigrationConfigurationTransaction.class); + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + DurableCutoverDraft draft = new DurableCutoverDraft( + OPERATION, MigrationTarget.MYSQL, ApplyMode.EXTERNAL_APPLY, + CREATED, STARTED, null); + DurableCutoverPreparation preparation = new DurableCutoverPreparation( + draft, store, configuration); + + try (SecretValue borrowed = password()) { + preparation.prepare(context(), MYSQL, borrowed); + } + + assertThat(store.find(OPERATION)).contains(running(ApplyMode.EXTERNAL_APPLY)); + verifyNoInteractions(configuration); + } + + @Test + void existingExactRunningIsConfirmedButNeverRestagedOrAllowedToRecopy() { + ManagedMigrationConfigurationTransaction configuration = mock( + ManagedMigrationConfigurationTransaction.class); + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + store.createOrConfirm(clean(ApplyMode.MANAGED_WRITE)); + store.compareAndTransitionOrConfirm( + OPERATION, MigrationOperationState.PENDING, running(ApplyMode.MANAGED_WRITE)); + DurableCutoverPreparation preparation = managed(store, configuration); + + try (SecretValue borrowed = password()) { + assertPreparationError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> preparation.prepare(context(), MYSQL, borrowed)); + } + + verifyNoInteractions(configuration); + assertThat(store.find(OPERATION)).contains(running(ApplyMode.MANAGED_WRITE)); + } + + @Test + void progressedRunningIsConfirmedButNeverRestagedOrAllowedToRecopy() { + ManagedMigrationConfigurationTransaction configuration = mock( + ManagedMigrationConfigurationTransaction.class); + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + MigrationOperationSnapshot running = running(ApplyMode.MANAGED_WRITE); + MigrationOperationSnapshot progressed = snapshot( + ApplyMode.MANAGED_WRITE, MigrationOperationState.RUNNING, MigrationStage.COPYING, + 35, STARTED, null, null, 1000); + store.createOrConfirm(clean(ApplyMode.MANAGED_WRITE)); + store.compareAndTransitionOrConfirm(OPERATION, MigrationOperationState.PENDING, running); + store.compareAndTransitionOrConfirm(OPERATION, MigrationOperationState.RUNNING, progressed); + + try (SecretValue borrowed = password()) { + assertPreparationError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> managed(store, configuration).prepare(context(), MYSQL, borrowed)); + } + + verifyNoInteractions(configuration); + assertThat(store.find(OPERATION)).contains(progressed); + } + + @Test + void terminalReplayIsDurablyConfirmedAndReportedAsStopWithoutRestaging() { + ManagedMigrationConfigurationTransaction configuration = mock( + ManagedMigrationConfigurationTransaction.class); + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + MigrationOperationSnapshot terminal = failed(SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED); + store.createOrConfirm(clean(ApplyMode.MANAGED_WRITE)); + store.compareAndTransitionOrConfirm(OPERATION, MigrationOperationState.PENDING, terminal); + + try (SecretValue borrowed = password()) { + assertPreparationError(SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED, + () -> managed(store, configuration).prepare(context(), MYSQL, borrowed)); + } + + verifyNoInteractions(configuration); + assertThat(store.find(OPERATION)).contains(terminal); + } + + @Test + void concurrentPreparationsAllowOnlyTheExactTransitionWinnerToContinue() throws Exception { + ManagedMigrationConfigurationTransaction configuration = mock( + ManagedMigrationConfigurationTransaction.class); + CountDownLatch bothStaging = new CountDownLatch(2); + CountDownLatch releaseStage = new CountDownLatch(1); + when(configuration.stageMetadataTarget(eq(OPERATION), eq(GENERATION), eq(IDENTITY), + same(MYSQL), any(SecretValue.class))).thenAnswer(invocation -> { + bothStaging.countDown(); + assertThat(bothStaging.await(5, SECONDS)).isTrue(); + assertThat(releaseStage.await(5, SECONDS)).isTrue(); + return staged(StageOutcome.ALREADY_STAGED); + }); + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + DurableCutoverPreparation first = managed(store, configuration); + DurableCutoverPreparation second = managed(store, configuration); + + try (ExecutorService workers = Executors.newFixedThreadPool(2)) { + Future firstResult = workers.submit(() -> invoke(first)); + Future secondResult = workers.submit(() -> invoke(second)); + assertThat(bothStaging.await(5, SECONDS)).isTrue(); + releaseStage.countDown(); + + assertThat(Arrays.asList( + firstResult.get(5, SECONDS), secondResult.get(5, SECONDS))) + .satisfiesExactlyInAnyOrder( + result -> assertThat(result).isNull(), + result -> assertThat(result) + .isInstanceOfSatisfying( + DurableCutoverPreparationException.class, + failure -> assertThat(failure.errorCode()) + .isEqualTo(SetupErrorCode.CONFIG_RECOVERY_REQUIRED))); + } finally { + releaseStage.countDown(); + } + assertThat(store.find(OPERATION)).contains(running(ApplyMode.MANAGED_WRITE)); + } + + @Test + void identityAndGenerationConflictsNeverTouchManagedConfiguration() throws Exception { + ManagedMigrationConfigurationTransaction configuration = mock( + ManagedMigrationConfigurationTransaction.class); + when(configuration.stageMetadataTarget(eq(OPERATION), eq(GENERATION), eq(IDENTITY), + same(MYSQL), any(SecretValue.class))) + .thenReturn(new MetadataTargetStageResult(StageOutcome.RECOVERY_REQUIRED, Optional.empty())); + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + DurableCutoverPreparation original = managed(store, configuration); + try (SecretValue borrowed = password()) { + assertPreparationError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> original.prepare(context(), MYSQL, borrowed)); + } + + DurableCutoverPreparation conflicting = new DurableCutoverPreparation( + new DurableCutoverDraft(OPERATION, MigrationTarget.MYSQL, ApplyMode.MANAGED_WRITE, + CREATED, STARTED, "other-generation"), store, configuration); + try (SecretValue borrowed = password()) { + assertThatThrownBy(() -> conflicting.prepare(context(), MYSQL, borrowed)) + .isInstanceOfSatisfying(DurableCutoverPreparationException.class, failure -> + assertThat(failure.errorCode()).isEqualTo(SetupErrorCode.OPERATION_CONFLICT)); + } + + verify(configuration, never()).stageMetadataTarget( + eq(OPERATION), eq("other-generation"), eq(IDENTITY), same(MYSQL), any(SecretValue.class)); + } + + @Test + void targetKindMismatchIsRejectedBeforeJournalOrCandidateAndDraftHasNoSecretSurface() { + ManagedMigrationConfigurationTransaction configuration = mock( + ManagedMigrationConfigurationTransaction.class); + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + MetadataDatabaseSettings postgres = new MetadataDatabaseSettings( + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db.example/hertzbeat", "migration"); + + try (SecretValue borrowed = password(); SecretValue expected = password()) { + assertPreparationError(SetupErrorCode.OPERATION_CONFLICT, + () -> managed(store, configuration).prepare(context(), postgres, borrowed)); + assertThat(borrowed).isEqualTo(expected); + } + + assertThat(store.history()).isEmpty(); + verifyNoInteractions(configuration); + List> fieldTypes = Arrays.stream(DurableCutoverPreparation.class.getDeclaredFields()) + .map(Field::getType) + .toList(); + assertThat(fieldTypes).doesNotContain(SecretValue.class); + assertThat(fieldTypes).doesNotContain(MetadataDatabaseSettings.class); + assertThatThrownBy(() -> new DurableCutoverDraft( + OPERATION, MigrationTarget.MYSQL, ApplyMode.MANAGED_WRITE, + STARTED, CREATED, GENERATION)).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> new DurableCutoverDraft( + OPERATION, MigrationTarget.MYSQL, ApplyMode.MANAGED_WRITE, + CREATED, STARTED, null)).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> new DurableCutoverDraft( + OPERATION, MigrationTarget.MYSQL, ApplyMode.EXTERNAL_APPLY, + CREATED, STARTED, GENERATION)).isInstanceOf(IllegalArgumentException.class); + } + + @Test + void fatalRemainsPrimaryAndBorrowedSecretRemainsOwnedByCaller() throws Exception { + ManagedMigrationConfigurationTransaction configuration = mock( + ManagedMigrationConfigurationTransaction.class); + AssertionError fatal = new AssertionError("fatal candidate write"); + when(configuration.stageMetadataTarget(eq(OPERATION), eq(GENERATION), eq(IDENTITY), + same(MYSQL), any(SecretValue.class))) + .thenThrow(fatal); + DurableCutoverPreparation preparation = managed( + new FileMigrationOperationStore(root), configuration); + + try (SecretValue borrowed = password(); SecretValue expected = password()) { + assertThatThrownBy(() -> preparation.prepare(context(), MYSQL, borrowed)).isSameAs(fatal); + assertThat(borrowed).isEqualTo(expected); + } + } + + private void assertTerminalOutcome(StageOutcome outcome, SetupErrorCode errorCode) throws Exception { + Path operationRoot = root.resolve(outcome.name().toLowerCase()); + ManagedMigrationConfigurationTransaction configuration = mock( + ManagedMigrationConfigurationTransaction.class); + when(configuration.stageMetadataTarget(eq(OPERATION), eq(GENERATION), eq(IDENTITY), + same(MYSQL), any(SecretValue.class))) + .thenReturn(new MetadataTargetStageResult(outcome, Optional.empty())); + FileMigrationOperationStore store = new FileMigrationOperationStore(operationRoot); + DurableCutoverPreparation preparation = managed(store, configuration); + + try (SecretValue borrowed = password()) { + assertPreparationError(errorCode, () -> preparation.prepare(context(), MYSQL, borrowed)); + assertPreparationError(errorCode, () -> preparation.prepare(context(), MYSQL, borrowed)); + } + + assertThat(store.find(OPERATION)).contains(failed(errorCode)); + verify(configuration).stageMetadataTarget( + eq(OPERATION), eq(GENERATION), eq(IDENTITY), same(MYSQL), any(SecretValue.class)); + } + + private static DurableCutoverPreparation managed( + FileMigrationOperationStore store, + ManagedMigrationConfigurationTransaction configuration) { + DurableCutoverDraft draft = new DurableCutoverDraft( + OPERATION, MigrationTarget.MYSQL, ApplyMode.MANAGED_WRITE, + CREATED, STARTED, GENERATION); + return new DurableCutoverPreparation(draft, store, configuration); + } + + private static MetadataTargetStageResult staged(StageOutcome outcome) { + return new MetadataTargetStageResult( + outcome, Optional.of(new CandidateRef(OPERATION, GENERATION))); + } + + private static RetainedCutoverPreparationContext context() { + return new RetainedCutoverPreparationContext(OPERATION, IDENTITY); + } + + private static SecretValue password() { + return SecretValue.of("target-password"); + } + + private static void assertPreparationError(SetupErrorCode code, Runnable action) { + assertThatThrownBy(action::run) + .isInstanceOfSatisfying(DurableCutoverPreparationException.class, failure -> { + assertThat(failure.errorCode()).isEqualTo(code); + assertThat(failure).hasNoCause(); + }); + } + + private static MigrationOperationSnapshot clean(ApplyMode mode) { + return snapshot(mode, MigrationOperationState.PENDING, MigrationStage.QUEUED, + null, null, null, 1000); + } + + private static MigrationOperationSnapshot blocked(ApplyMode mode) { + return snapshot(mode, MigrationOperationState.PENDING, MigrationStage.QUEUED, + null, null, SetupErrorCode.CONFIG_RECOVERY_REQUIRED, 0); + } + + private static MigrationOperationSnapshot running(ApplyMode mode) { + return snapshot(mode, MigrationOperationState.RUNNING, MigrationStage.COPYING, + STARTED, null, null, 1000); + } + + private static MigrationOperationSnapshot failed(SetupErrorCode code) { + return snapshot(ApplyMode.MANAGED_WRITE, MigrationOperationState.FAILED, + MigrationStage.FAILED, null, STARTED, code, 0); + } + + private static MigrationOperationSnapshot snapshot( + ApplyMode mode, MigrationOperationState state, MigrationStage stage, + Instant startedAt, Instant completedAt, SetupErrorCode error, long pollMillis) { + return snapshot(mode, state, stage, 0, + startedAt, completedAt, error, pollMillis); + } + + private static MigrationOperationSnapshot snapshot( + ApplyMode mode, MigrationOperationState state, MigrationStage stage, int progress, + Instant startedAt, Instant completedAt, SetupErrorCode error, long pollMillis) { + return new MigrationOperationSnapshot(OPERATION, state, MigrationTarget.MYSQL, mode, stage, + progress, CREATED, startedAt, completedAt, + VerificationState.PENDING, error, null, pollMillis, false, false, false, + IDENTITY, mode == ApplyMode.MANAGED_WRITE ? GENERATION : null); + } + + private static Throwable invoke(DurableCutoverPreparation preparation) { + try (SecretValue borrowed = password()) { + preparation.prepare(context(), MYSQL, borrowed); + return null; + } catch (Throwable failure) { + return failure; + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStoreExactTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStoreExactTest.java index 3355162d7f..28e8c1c208 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStoreExactTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStoreExactTest.java @@ -23,6 +23,7 @@ import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.Verification import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; import org.apache.hertzbeat.manager.setup.security.CommittedSetupFileDurabilityException; +import org.apache.hertzbeat.manager.setup.workflow.FileMigrationOperationStore.ExactTransitionDisposition; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -43,10 +44,12 @@ class FileMigrationOperationStoreExactTest { assertThat(store.createOrConfirm(pending)).isEqualTo(pending); assertThat(store.createOrConfirm(pending)).isEqualTo(pending); - assertThat(store.compareAndTransitionOrConfirm( - pending.operationId(), MigrationOperationState.PENDING, running)).isEqualTo(running); - assertThat(store.compareAndTransitionOrConfirm( - pending.operationId(), MigrationOperationState.PENDING, running)).isEqualTo(running); + assertThat(store.compareAndTransitionOrConfirmDisposition( + pending.operationId(), MigrationOperationState.PENDING, running)) + .isEqualTo(ExactTransitionDisposition.TRANSITIONED); + assertThat(store.compareAndTransitionOrConfirmDisposition( + pending.operationId(), MigrationOperationState.PENDING, running)) + .isEqualTo(ExactTransitionDisposition.ALREADY_CONFIRMED); } @Test diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverPreparationTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverPreparationTest.java index 6d30b2a066..07dd7736e7 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverPreparationTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverPreparationTest.java @@ -53,7 +53,7 @@ class RetainedCutoverPreparationTest { doAnswer(invocation -> { observed.set(invocation.getArgument(0)); return null; - }).when(preparation).prepare(any()); + }).when(preparation).prepare(any(), same(TARGET), same(fixture.password)); fixture.execute(preparation); @@ -62,17 +62,17 @@ class RetainedCutoverPreparationTest { InOrder order = inOrder(fixture.factory, fixture.provisionLease, preparation, fixture.provisioner); order.verify(fixture.factory).acquire(same(TARGET), same(fixture.password), anyDeadline()); order.verify(fixture.provisionLease).targetIdentityHash(); - order.verify(preparation).prepare(observed.get()); + order.verify(preparation).prepare(observed.get(), TARGET, fixture.password); order.verify(fixture.provisionLease).withConnection(any()); order.verify(fixture.provisioner).provision( same(fixture.provisionConnection), eq(MetadataDatabaseKind.MYSQL), anyDeadline()); - verify(preparation).prepare(any()); + verify(preparation).prepare(any(), same(TARGET), same(fixture.password)); } @Test void elapsedRootDeadlineAfterPreparationPreventsProvision() { Fixture fixture = new Fixture(); - RetainedCutoverPreparation preparation = context -> fixture.ticker.set(100); + RetainedCutoverPreparation preparation = (context, target, password) -> fixture.ticker.set(100); assertThatThrownBy(() -> fixture.execute(preparation)) .isInstanceOfSatisfying(MetadataMigrationException.class, failure -> @@ -129,7 +129,7 @@ class RetainedCutoverPreparationTest { Fixture stable = new Fixture(); MetadataMigrationException expected = new MetadataMigrationException( MetadataMigrationErrorCode.VERIFICATION); - RetainedCutoverPreparation stableFailure = context -> { + RetainedCutoverPreparation stableFailure = (context, target, password) -> { throw expected; }; @@ -138,7 +138,7 @@ class RetainedCutoverPreparationTest { verifyNoInteractions(stable.provisioner, stable.maintenance, stable.executor); Fixture unexpected = new Fixture(); - RetainedCutoverPreparation privateFailure = context -> { + RetainedCutoverPreparation privateFailure = (context, target, password) -> { throw new IllegalStateException("private preparation diagnostic"); }; assertThatThrownBy(() -> unexpected.execute(privateFailure)) @@ -149,12 +149,26 @@ class RetainedCutoverPreparationTest { verifyNoInteractions(unexpected.provisioner, unexpected.maintenance, unexpected.executor); } + @Test + void durableStopCodeIsPreservedWithoutAllowingProvision() { + Fixture fixture = new Fixture(); + DurableCutoverPreparationException expected = new DurableCutoverPreparationException( + org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + RetainedCutoverPreparation stop = (context, target, password) -> { + throw expected; + }; + + assertThatThrownBy(() -> fixture.execute(stop)).isSameAs(expected); + verify(fixture.provisionLease).close(); + verifyNoInteractions(fixture.provisioner, fixture.maintenance, fixture.executor); + } + @Test void fatalPreparationRemainsPrimaryAcrossExactCloseRetry() { Fixture fixture = new Fixture(); AssertionError fatal = new AssertionError("preparation fatal"); RetainedCutoverPreparation preparation = mock(RetainedCutoverPreparation.class); - doThrow(fatal).when(preparation).prepare(any()); + doThrow(fatal).when(preparation).prepare(any(), same(TARGET), same(fixture.password)); doThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED)) .doNothing().when(fixture.provisionLease).close(); @@ -165,7 +179,7 @@ class RetainedCutoverPreparationTest { assertThatThrownBy(() -> fixture.coordinator.retryRelease(OPERATION, Duration.ofSeconds(1))) .isSameAs(fatal); - verify(preparation).prepare(any()); + verify(preparation).prepare(any(), same(TARGET), same(fixture.password)); verify(fixture.provisionLease, times(2)).close(); verifyNoInteractions(fixture.provisioner, fixture.maintenance, fixture.executor); } @@ -173,7 +187,8 @@ class RetainedCutoverPreparationTest { @Test void preparationInterruptIsClearedForCloseAndRestoredForCaller() { Fixture fixture = new Fixture(); - RetainedCutoverPreparation preparation = context -> Thread.currentThread().interrupt(); + RetainedCutoverPreparation preparation = + (context, target, password) -> Thread.currentThread().interrupt(); doAnswer(invocation -> { assertThat(Thread.currentThread().isInterrupted()).isFalse(); return null; @@ -194,7 +209,7 @@ class RetainedCutoverPreparationTest { Fixture fixture = new Fixture(); AtomicReference sameOperation = new AtomicReference<>(); AtomicReference foreignOperation = new AtomicReference<>(); - RetainedCutoverPreparation preparation = context -> { + RetainedCutoverPreparation preparation = (context, target, password) -> { capture(sameOperation, () -> fixture.execute(RetainedCutoverPreparation.NO_OP)); capture(foreignOperation, () -> fixture.coordinator.execute( "operation-b", TARGET, fixture.password, TIMEOUT, From f753f21c2259f1af5dc4044d969131d1f9d422b0 Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 15:20:42 +0800 Subject: [PATCH 56/71] Handoff retained metadata copy journal --- .../DurableRetainedCopyJournalHandoff.java | 97 ++++++ .../workflow/RetainedCopyJournalContext.java | 23 ++ .../RetainedCopyJournalDisposition.java | 14 + .../workflow/RetainedCopyJournalHandoff.java | 17 ++ .../RetainedCopyJournalHandoffException.java | 27 ++ .../RetainedCopyJournalSnapshots.java | 92 ++++++ .../workflow/RetainedCutoverCoordinator.java | 44 ++- .../workflow/RetainedCutoverOutcome.java | 1 + .../setup/workflow/RetainedCutoverState.java | 49 ++- ...DurableRetainedCopyJournalHandoffTest.java | 283 ++++++++++++++++++ .../RetainedCutoverCoordinatorTest.java | 102 ++++++- .../RetainedCutoverFactoryRaceTest.java | 3 +- .../workflow/RetainedCutoverFailureTest.java | 9 +- .../RetainedCutoverJournalHandoffTest.java | 197 ++++++++++++ .../RetainedCutoverLifecycleTest.java | 6 +- .../RetainedCutoverPreparationTest.java | 6 +- 16 files changed, 945 insertions(+), 25 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableRetainedCopyJournalHandoff.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCopyJournalContext.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCopyJournalDisposition.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCopyJournalHandoff.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCopyJournalHandoffException.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCopyJournalSnapshots.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DurableRetainedCopyJournalHandoffTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverJournalHandoffTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableRetainedCopyJournalHandoff.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableRetainedCopyJournalHandoff.java new file mode 100644 index 0000000000..a2628de0a1 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableRetainedCopyJournalHandoff.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.workflow.FileMigrationOperationStore.ExactTransitionDisposition; + +/** Durably records verification and cutover readiness after an already completed metadata copy. */ +final class DurableRetainedCopyJournalHandoff implements RetainedCopyJournalHandoff { + + private final DurableCutoverDraft draft; + private final FileMigrationOperationStore store; + + DurableRetainedCopyJournalHandoff( + DurableCutoverDraft draft, FileMigrationOperationStore store) { + this.draft = Objects.requireNonNull(draft, "draft"); + this.store = Objects.requireNonNull(store, "store"); + } + + @Override + public RetainedCopyJournalDisposition handoff(RetainedCopyJournalContext context) { + Objects.requireNonNull(context, "context"); + try { + return handoffDurably(context); + } catch (MigrationOperationStoreException failure) { + throw new RetainedCopyJournalHandoffException(failure.errorCode()); + } catch (RetainedCopyJournalHandoffException failure) { + throw failure; + } catch (RuntimeException failure) { + throw new RetainedCopyJournalHandoffException(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + } + + private RetainedCopyJournalDisposition handoffDurably(RetainedCopyJournalContext context) { + MigrationOperationSnapshot current = store.find(context.operationId()) + .orElseThrow(() -> new RetainedCopyJournalHandoffException( + SetupErrorCode.OPERATION_NOT_FOUND)); + if (!exactIdentity(current, context)) { + throw new RetainedCopyJournalHandoffException(SetupErrorCode.OPERATION_CONFLICT); + } + RetainedCopyJournalSnapshots snapshots = new RetainedCopyJournalSnapshots(current); + if (snapshots.finalState()) { + confirmExact(current); + return RetainedCopyJournalDisposition.ALREADY_CONFIRMED; + } + MigrationOperationSnapshot verifying; + if (snapshots.copying()) { + verifying = snapshots.verifyingSnapshot(); + transition(current, verifying); + } else if (snapshots.verifying()) { + verifying = current; + confirmExact(current); + } else { + throw new RetainedCopyJournalHandoffException(SetupErrorCode.OPERATION_CONFLICT); + } + MigrationOperationSnapshot ready = new RetainedCopyJournalSnapshots(verifying).finalSnapshot(); + ExactTransitionDisposition disposition = store.compareAndTransitionOrConfirmDisposition( + context.operationId(), MigrationOperationState.RUNNING, ready); + return disposition == ExactTransitionDisposition.TRANSITIONED + ? RetainedCopyJournalDisposition.TRANSITIONED + : RetainedCopyJournalDisposition.ALREADY_CONFIRMED; + } + + private boolean exactIdentity( + MigrationOperationSnapshot current, RetainedCopyJournalContext context) { + return draft.operationId().equals(context.operationId()) + && current.operationId().equals(draft.operationId()) + && current.target() == draft.target() + && current.applyMode() == draft.applyMode() + && current.createdAt().equals(draft.createdAt()) + && Objects.equals(current.startedAt(), draft.startedAt()) + && current.targetIdentityHash().equals(context.targetIdentityHash()) + && Objects.equals( + current.managedCandidateGeneration(), draft.candidateGeneration()); + } + + private void transition( + MigrationOperationSnapshot current, MigrationOperationSnapshot replacement) { + store.compareAndTransitionOrConfirmDisposition( + current.operationId(), MigrationOperationState.RUNNING, replacement); + } + + private void confirmExact(MigrationOperationSnapshot current) { + ExactTransitionDisposition disposition = store.compareAndTransitionOrConfirmDisposition( + current.operationId(), current.state(), current); + if (disposition != ExactTransitionDisposition.ALREADY_CONFIRMED) { + throw new RetainedCopyJournalHandoffException(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCopyJournalContext.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCopyJournalContext.java new file mode 100644 index 0000000000..8677956b08 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCopyJournalContext.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import org.apache.hertzbeat.manager.setup.api.OperationIdValidator; + +/** Secret-free identity of the retained copy whose journal handoff is running. */ +record RetainedCopyJournalContext(String operationId, String targetIdentityHash) { + + RetainedCopyJournalContext { + if (!OperationIdValidator.isSafe(operationId)) { + throw new IllegalArgumentException("Invalid operation id"); + } + if (targetIdentityHash == null || !targetIdentityHash.matches("[0-9a-f]{64}")) { + throw new IllegalArgumentException("Invalid target identity"); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCopyJournalDisposition.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCopyJournalDisposition.java new file mode 100644 index 0000000000..d57925b777 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCopyJournalDisposition.java @@ -0,0 +1,14 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +/** Reports whether this invocation durably won the final journal transition. */ +enum RetainedCopyJournalDisposition { + TRANSITIONED, + ALREADY_CONFIRMED +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCopyJournalHandoff.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCopyJournalHandoff.java new file mode 100644 index 0000000000..4b8b1c177e --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCopyJournalHandoff.java @@ -0,0 +1,17 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +/** Persists the credential-free journal outcome for one retained successful copy. */ +@FunctionalInterface +interface RetainedCopyJournalHandoff { + + RetainedCopyJournalHandoff NO_OP = context -> RetainedCopyJournalDisposition.TRANSITIONED; + + RetainedCopyJournalDisposition handoff(RetainedCopyJournalContext context); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCopyJournalHandoffException.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCopyJournalHandoffException.java new file mode 100644 index 0000000000..13be5df5f4 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCopyJournalHandoffException.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Stable, cause-free failure that leaves the exact maintenance fence retained. */ +final class RetainedCopyJournalHandoffException extends RuntimeException { + + private final SetupErrorCode errorCode; + + RetainedCopyJournalHandoffException(SetupErrorCode errorCode) { + super("Retained copy journal handoff failed: " + + Objects.requireNonNull(errorCode, "errorCode").value()); + this.errorCode = errorCode; + } + + SetupErrorCode errorCode() { + return errorCode; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCopyJournalSnapshots.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCopyJournalSnapshots.java new file mode 100644 index 0000000000..7b773b473b --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCopyJournalSnapshots.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; + +/** Builds and recognizes the exact secret-free journal shapes after a successful copy. */ +final class RetainedCopyJournalSnapshots { + + private static final long ACTIVE_POLL_MILLIS = 1000; + + private final MigrationOperationSnapshot source; + + RetainedCopyJournalSnapshots(MigrationOperationSnapshot source) { + this.source = source; + } + + boolean copying() { + return source.state() == MigrationOperationState.RUNNING + && source.stage() == MigrationStage.COPYING + && source.progressPercent() < 100 + && source.startedAt() != null + && source.completedAt() == null + && source.verificationState() == VerificationState.PENDING + && source.errorCode() == null + && source.rollbackOrigin() == null + && !source.activationAvailable() + && !source.restartRequired() + && !source.externalApplyRequired(); + } + + boolean verifying() { + return source.state() == MigrationOperationState.RUNNING + && source.stage() == MigrationStage.VERIFYING + && source.progressPercent() == 100 + && source.startedAt() != null + && source.completedAt() == null + && source.verificationState() == VerificationState.RUNNING + && source.errorCode() == null + && source.rollbackOrigin() == null + && !source.activationAvailable() + && !source.restartRequired() + && !source.externalApplyRequired(); + } + + boolean finalState() { + return source.applyMode() == ApplyMode.MANAGED_WRITE + ? source.state() == MigrationOperationState.READY_TO_ACTIVATE + && source.stage() == MigrationStage.READY_TO_ACTIVATE + && source.activationAvailable() && !source.externalApplyRequired() + : source.state() == MigrationOperationState.AWAITING_EXTERNAL_APPLY + && source.stage() == MigrationStage.AWAITING_EXTERNAL_APPLY + && !source.activationAvailable() && source.externalApplyRequired(); + } + + MigrationOperationSnapshot verifyingSnapshot() { + return snapshot(MigrationOperationState.RUNNING, MigrationStage.VERIFYING, + VerificationState.RUNNING, ACTIVE_POLL_MILLIS, false, false); + } + + MigrationOperationSnapshot finalSnapshot() { + boolean managed = source.applyMode() == ApplyMode.MANAGED_WRITE; + return snapshot( + managed ? MigrationOperationState.READY_TO_ACTIVATE + : MigrationOperationState.AWAITING_EXTERNAL_APPLY, + managed ? MigrationStage.READY_TO_ACTIVATE + : MigrationStage.AWAITING_EXTERNAL_APPLY, + VerificationState.SUCCEEDED, 0, managed, !managed); + } + + private MigrationOperationSnapshot snapshot( + MigrationOperationState state, + MigrationStage stage, + VerificationState verification, + long pollMillis, + boolean activationAvailable, + boolean externalApplyRequired) { + return new MigrationOperationSnapshot( + source.operationId(), state, source.target(), source.applyMode(), stage, 100, + source.createdAt(), source.startedAt(), null, verification, null, null, + pollMillis, activationAvailable, false, externalApplyRequired, + source.targetIdentityHash(), source.managedCandidateGeneration()); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java index 2765621f1f..ce605a3f96 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java @@ -14,6 +14,7 @@ import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceException; import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceLease; import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceOrchestrator; import org.apache.hertzbeat.manager.setup.api.OperationIdValidator; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; import org.apache.hertzbeat.manager.setup.config.SecretValue; @@ -49,10 +50,11 @@ final class RetainedCutoverCoordinator { SecretValue borrowedPassword, Duration timeout, MetadataMigrationProgressSink progress, - RetainedCutoverPreparation preparation) { - requireRequest(operationId, target, borrowedPassword, timeout, progress, preparation); + RetainedCutoverPreparation preparation, + RetainedCopyJournalHandoff handoff) { + requireRequest(operationId, target, borrowedPassword, timeout, progress, preparation, handoff); JdbcMetadataMigrationDeadline deadline = JdbcMetadataMigrationDeadline.start(timeout, ticker); - RetainedCutoverState.Execution execution = state.reserve(operationId); + RetainedCutoverState.Execution execution = state.reserve(operationId, handoff); TargetJdbcConnectionLease provisionLease = acquire(execution, target, borrowedPassword, deadline); String provisionIdentity = targetIdentity(execution, provisionLease, deadline); execution.targetIdentityHash(provisionIdentity); @@ -111,6 +113,11 @@ final class RetainedCutoverCoordinator { return finish(execution, execution.release(), deadline); } + RetainedCutoverResult retryHandoff(String operationId) { + requireOperationId(operationId); + return runHandoff(state.claimPendingHandoff(operationId)); + } + private TargetJdbcConnectionLease acquire( RetainedCutoverState.Execution execution, MetadataDatabaseSettings target, @@ -192,16 +199,35 @@ final class RetainedCutoverCoordinator { restoreInterrupt(interrupted | Thread.interrupted()); } if (advance == RetainedCutoverRelease.Advance.RETAINED) { - return retain(execution, release.takeRetainedMaintenance()); + state.beginHandoff(execution, release.takeRetainedMaintenance()); + return runHandoff(execution); } state.clear(execution); release.outcome().replay(); return execution.result(RetainedCutoverResult.Status.RELEASED); } - private RetainedCutoverResult retain( - RetainedCutoverState.Execution execution, MigrationMaintenanceLease maintenanceLease) { - return state.retain(execution, maintenanceLease); + private RetainedCutoverResult runHandoff(RetainedCutoverState.Execution execution) { + try { + if (Thread.currentThread().isInterrupted()) { + throw new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT); + } + RetainedCopyJournalDisposition disposition = Objects.requireNonNull( + execution.handoff().handoff(execution.handoffContext()), "handoff disposition"); + if (Thread.currentThread().isInterrupted()) { + throw new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT); + } + return state.completeHandoff(execution, disposition); + } catch (Error fatal) { + state.handoffPending(execution); + throw fatal; + } catch (RuntimeException failure) { + state.handoffPending(execution); + if (failure instanceof RetainedCopyJournalHandoffException handoffFailure) { + throw handoffFailure; + } + throw new RetainedCopyJournalHandoffException(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } } private void releasePending( @@ -219,13 +245,15 @@ final class RetainedCutoverCoordinator { SecretValue password, Duration timeout, MetadataMigrationProgressSink progress, - RetainedCutoverPreparation preparation) { + RetainedCutoverPreparation preparation, + RetainedCopyJournalHandoff handoff) { requireOperationId(operationId); Objects.requireNonNull(target, "target"); Objects.requireNonNull(password, "password"); Objects.requireNonNull(timeout, "timeout"); Objects.requireNonNull(progress, "progress"); Objects.requireNonNull(preparation, "preparation"); + Objects.requireNonNull(handoff, "handoff"); } private static void requireOperationId(String operationId) { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverOutcome.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverOutcome.java index 37a9aff705..d837fd3e25 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverOutcome.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverOutcome.java @@ -48,6 +48,7 @@ final class RetainedCutoverOutcome { || failure instanceof TargetSchemaProvisioningException || failure instanceof MigrationMaintenanceException || failure instanceof DurableCutoverPreparationException + || failure instanceof RetainedCopyJournalHandoffException || failure instanceof RetainedCutoverException) { return stable((RuntimeException) failure); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverState.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverState.java index 93646efdac..c42ab677b3 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverState.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverState.java @@ -16,11 +16,11 @@ final class RetainedCutoverState { private Execution active; - synchronized Execution reserve(String operationId) { + synchronized Execution reserve(String operationId, RetainedCopyJournalHandoff handoff) { if (active != null) { throw MigrationMaintenanceException.operationConflict(); } - active = new Execution(operationId); + active = new Execution(operationId, handoff); return active; } @@ -44,6 +44,12 @@ final class RetainedCutoverState { return execution; } + synchronized Execution claimPendingHandoff(String operationId) { + Execution execution = require(operationId, Phase.HANDOFF_PENDING); + execution.phase = Phase.HANDOFFING; + return execution; + } + synchronized void releasePending(Execution execution, RetainedCutoverRelease release) { if (active == execution) { execution.release = release; @@ -51,15 +57,34 @@ final class RetainedCutoverState { } } - synchronized RetainedCutoverResult retain( + synchronized void beginHandoff( Execution execution, MigrationMaintenanceLease maintenanceLease) { - if (active != execution) { + if (active != execution + || (execution.phase != Phase.EXECUTING && execution.phase != Phase.RELEASING)) { throw MigrationMaintenanceException.operationConflict(); } execution.maintenanceLease = Objects.requireNonNull(maintenanceLease, "maintenanceLease"); execution.release = null; + execution.phase = Phase.HANDOFFING; + } + + synchronized RetainedCutoverResult completeHandoff( + Execution execution, RetainedCopyJournalDisposition disposition) { + if (active != execution || execution.phase != Phase.HANDOFFING) { + throw MigrationMaintenanceException.operationConflict(); + } execution.phase = Phase.RETAINED; - return execution.result(RetainedCutoverResult.Status.RETAINED_SUCCESS); + RetainedCutoverResult.Status status = disposition == RetainedCopyJournalDisposition.TRANSITIONED + ? RetainedCutoverResult.Status.RETAINED_SUCCESS + : RetainedCutoverResult.Status.ALREADY_RETAINED; + return execution.result(status); + } + + synchronized void handoffPending(Execution execution) { + if (active != execution || execution.phase != Phase.HANDOFFING) { + throw MigrationMaintenanceException.operationConflict(); + } + execution.phase = Phase.HANDOFF_PENDING; } synchronized void clear(Execution execution) { @@ -80,13 +105,15 @@ final class RetainedCutoverState { static final class Execution { private final String operationId; + private final RetainedCopyJournalHandoff handoff; private String targetIdentityHash; private MigrationMaintenanceLease maintenanceLease; private RetainedCutoverRelease release; private Phase phase = Phase.EXECUTING; - private Execution(String operationId) { + private Execution(String operationId, RetainedCopyJournalHandoff handoff) { this.operationId = operationId; + this.handoff = Objects.requireNonNull(handoff, "handoff"); } void targetIdentityHash(String targetIdentityHash) { @@ -97,6 +124,14 @@ final class RetainedCutoverState { return release; } + RetainedCopyJournalContext handoffContext() { + return new RetainedCopyJournalContext(operationId, targetIdentityHash); + } + + RetainedCopyJournalHandoff handoff() { + return handoff; + } + RetainedCutoverResult result(RetainedCutoverResult.Status status) { return new RetainedCutoverResult(operationId, targetIdentityHash, status); } @@ -104,6 +139,8 @@ final class RetainedCutoverState { private enum Phase { EXECUTING, + HANDOFFING, + HANDOFF_PENDING, RETAINED, RELEASING, RELEASE_PENDING diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DurableRetainedCopyJournalHandoffTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DurableRetainedCopyJournalHandoffTest.java new file mode 100644 index 0000000000..ca85b007d1 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DurableRetainedCopyJournalHandoffTest.java @@ -0,0 +1,283 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.security.CommittedSetupFileDurabilityException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class DurableRetainedCopyJournalHandoffTest { + + private static final String OPERATION = "operation-a"; + private static final String IDENTITY = "a".repeat(64); + private static final String GENERATION = "candidate-generation"; + private static final Instant CREATED = Instant.parse("2026-08-10T03:00:00Z"); + private static final Instant STARTED = CREATED.plusSeconds(1); + + @TempDir + private Path root; + + @Test + void managedCopyPersistsVerificationBeforeReady() { + FileMigrationOperationStore store = seeded(ApplyMode.MANAGED_WRITE, 35); + DurableRetainedCopyJournalHandoff handoff = handoff(store, ApplyMode.MANAGED_WRITE); + + assertThat(handoff.handoff(context())).isEqualTo(RetainedCopyJournalDisposition.TRANSITIONED); + + MigrationOperationSnapshot current = store.find(OPERATION).orElseThrow(); + assertThat(current.state()).isEqualTo(MigrationOperationState.READY_TO_ACTIVATE); + assertThat(current.stage()).isEqualTo(MigrationStage.READY_TO_ACTIVATE); + assertThat(current.verificationState()).isEqualTo(VerificationState.SUCCEEDED); + assertThat(current.progressPercent()).isEqualTo(100); + assertThat(current.activationAvailable()).isTrue(); + } + + @Test + void externalCopyPersistsAwaitingExternalApply() { + FileMigrationOperationStore store = seeded(ApplyMode.EXTERNAL_APPLY, 0); + + assertThat(handoff(store, ApplyMode.EXTERNAL_APPLY).handoff(context())) + .isEqualTo(RetainedCopyJournalDisposition.TRANSITIONED); + + MigrationOperationSnapshot current = store.find(OPERATION).orElseThrow(); + assertThat(current.state()).isEqualTo(MigrationOperationState.AWAITING_EXTERNAL_APPLY); + assertThat(current.externalApplyRequired()).isTrue(); + } + + @Test + void retryFromExactVerifyingOnlyPersistsTheFinalState() { + FileMigrationOperationStore store = seeded(ApplyMode.MANAGED_WRITE, 10); + MigrationOperationSnapshot copying = store.find(OPERATION).orElseThrow(); + store.compareAndTransitionOrConfirm( + OPERATION, MigrationOperationState.RUNNING, verifying(copying)); + + assertThat(handoff(store, ApplyMode.MANAGED_WRITE).handoff(context())) + .isEqualTo(RetainedCopyJournalDisposition.TRANSITIONED); + assertThat(store.find(OPERATION).orElseThrow().state()) + .isEqualTo(MigrationOperationState.READY_TO_ACTIVATE); + } + + @Test + void exactFinalReplayOnlyConfirmsDurability() { + FileMigrationOperationStore store = seeded(ApplyMode.MANAGED_WRITE, 10); + DurableRetainedCopyJournalHandoff handoff = handoff(store, ApplyMode.MANAGED_WRITE); + assertThat(handoff.handoff(context())).isEqualTo(RetainedCopyJournalDisposition.TRANSITIONED); + + assertThat(handoff.handoff(context())).isEqualTo(RetainedCopyJournalDisposition.ALREADY_CONFIRMED); + assertThat(store.find(OPERATION).orElseThrow().state()) + .isEqualTo(MigrationOperationState.READY_TO_ACTIVATE); + } + + @Test + void failedFinalPublishLeavesVerifyingForJournalOnlyRetry() { + FileMigrationOperationStore seed = seeded(ApplyMode.MANAGED_WRITE, 25); + MigrationOperationFilePublisher delegate = new MigrationOperationFilePublisher(root); + AtomicInteger publications = new AtomicInteger(); + FileMigrationOperationStore failing = new FileMigrationOperationStore(root, (target, content) -> { + if (publications.incrementAndGet() == 2) { + throw new IOException("private path"); + } + delegate.publish(target, content); + }); + DurableRetainedCopyJournalHandoff handoff = handoff(failing, ApplyMode.MANAGED_WRITE); + + assertSafeFailure(SetupErrorCode.CONFIG_WRITE_FAILED, () -> handoff.handoff(context())); + assertThat(seed.find(OPERATION).orElseThrow().stage()).isEqualTo(MigrationStage.VERIFYING); + + assertThat(handoff(seed, ApplyMode.MANAGED_WRITE).handoff(context())) + .isEqualTo(RetainedCopyJournalDisposition.TRANSITIONED); + } + + @Test + void failedVerificationPublishLeavesCopyingForJournalOnlyRetry() { + FileMigrationOperationStore seed = seeded(ApplyMode.MANAGED_WRITE, 25); + MigrationOperationFilePublisher delegate = new MigrationOperationFilePublisher(root); + AtomicInteger publications = new AtomicInteger(); + FileMigrationOperationStore failing = new FileMigrationOperationStore(root, (target, content) -> { + if (publications.incrementAndGet() == 1) { + throw new IOException("private path"); + } + delegate.publish(target, content); + }); + + assertSafeFailure(SetupErrorCode.CONFIG_WRITE_FAILED, + () -> handoff(failing, ApplyMode.MANAGED_WRITE).handoff(context())); + assertThat(seed.find(OPERATION).orElseThrow().stage()).isEqualTo(MigrationStage.COPYING); + + assertThat(handoff(seed, ApplyMode.MANAGED_WRITE).handoff(context())) + .isEqualTo(RetainedCopyJournalDisposition.TRANSITIONED); + } + + @Test + void committedUncertainVerificationIsConfirmedBeforeFinalState() { + seeded(ApplyMode.MANAGED_WRITE, 25); + MigrationOperationFilePublisher delegate = new MigrationOperationFilePublisher(root); + AtomicInteger publications = new AtomicInteger(); + FileMigrationOperationStore uncertain = new FileMigrationOperationStore(root, (target, content) -> { + delegate.publish(target, content); + if (publications.incrementAndGet() == 1) { + throw new CommittedSetupFileDurabilityException(); + } + }); + + assertThat(handoff(uncertain, ApplyMode.MANAGED_WRITE).handoff(context())) + .isEqualTo(RetainedCopyJournalDisposition.TRANSITIONED); + assertThat(uncertain.find(OPERATION).orElseThrow().state()) + .isEqualTo(MigrationOperationState.READY_TO_ACTIVATE); + assertThat(publications).hasValue(3); + } + + @Test + void committedUncertainFinalStateIsConfirmedBeforeSuccess() { + seeded(ApplyMode.MANAGED_WRITE, 25); + MigrationOperationFilePublisher delegate = new MigrationOperationFilePublisher(root); + AtomicInteger publications = new AtomicInteger(); + FileMigrationOperationStore uncertain = new FileMigrationOperationStore(root, (target, content) -> { + delegate.publish(target, content); + if (publications.incrementAndGet() == 2) { + throw new CommittedSetupFileDurabilityException(); + } + }); + + assertThat(handoff(uncertain, ApplyMode.MANAGED_WRITE).handoff(context())) + .isEqualTo(RetainedCopyJournalDisposition.TRANSITIONED); + assertThat(uncertain.find(OPERATION).orElseThrow().state()) + .isEqualTo(MigrationOperationState.READY_TO_ACTIVATE); + assertThat(publications).hasValue(3); + } + + @Test + void mismatchedIdentityAndUnexpectedStateFailClosed() { + FileMigrationOperationStore store = seeded(ApplyMode.MANAGED_WRITE, 10); + DurableRetainedCopyJournalHandoff handoff = handoff(store, ApplyMode.MANAGED_WRITE); + + assertSafeFailure(SetupErrorCode.OPERATION_CONFLICT, () -> handoff.handoff( + new RetainedCopyJournalContext(OPERATION, "b".repeat(64)))); + assertSafeFailure(SetupErrorCode.OPERATION_NOT_FOUND, () -> handoff.handoff( + new RetainedCopyJournalContext("missing-operation", IDENTITY))); + } + + @Test + void immutableDraftMismatchNeverMutatesTheJournal() { + FileMigrationOperationStore store = seeded(ApplyMode.MANAGED_WRITE, 10); + MigrationOperationSnapshot before = store.find(OPERATION).orElseThrow(); + byte[] encodedBefore = readJournal(); + DurableCutoverDraft wrongGeneration = new DurableCutoverDraft( + OPERATION, MigrationTarget.MYSQL, ApplyMode.MANAGED_WRITE, + CREATED, STARTED, "other-generation"); + DurableCutoverDraft wrongCreatedAt = new DurableCutoverDraft( + OPERATION, MigrationTarget.MYSQL, ApplyMode.MANAGED_WRITE, + CREATED.minusSeconds(1), STARTED, GENERATION); + DurableCutoverDraft wrongApplyMode = new DurableCutoverDraft( + OPERATION, MigrationTarget.MYSQL, ApplyMode.EXTERNAL_APPLY, + CREATED, STARTED, null); + + assertSafeFailure(SetupErrorCode.OPERATION_CONFLICT, + () -> new DurableRetainedCopyJournalHandoff(wrongGeneration, store).handoff(context())); + assertSafeFailure(SetupErrorCode.OPERATION_CONFLICT, + () -> new DurableRetainedCopyJournalHandoff(wrongCreatedAt, store).handoff(context())); + assertSafeFailure(SetupErrorCode.OPERATION_CONFLICT, + () -> new DurableRetainedCopyJournalHandoff(wrongApplyMode, store).handoff(context())); + + assertThat(store.find(OPERATION)).contains(before); + assertThat(readJournal()).containsExactly(encodedBefore); + } + + @Test + void handoffSurfaceIsCredentialFreeAndRedacted() { + DurableRetainedCopyJournalHandoff handoff = handoff( + seeded(ApplyMode.MANAGED_WRITE, 10), ApplyMode.MANAGED_WRITE); + + assertThat(handoff.toString()) + .doesNotContain("jdbc:", "password", "username", IDENTITY, GENERATION); + assertThat(DurableRetainedCopyJournalHandoff.class.getDeclaredFields()) + .allMatch(field -> field.getType() == FileMigrationOperationStore.class + || field.getType() == DurableCutoverDraft.class); + } + + private FileMigrationOperationStore seeded(ApplyMode applyMode, int progress) { + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + String generation = applyMode == ApplyMode.MANAGED_WRITE ? GENERATION : null; + MigrationOperationSnapshot pending = snapshot( + applyMode, MigrationOperationState.PENDING, MigrationStage.QUEUED, 0, + null, VerificationState.PENDING, false, false, generation); + store.create(pending); + store.compareAndTransition(OPERATION, MigrationOperationState.PENDING, snapshot( + applyMode, MigrationOperationState.RUNNING, MigrationStage.COPYING, progress, + STARTED, VerificationState.PENDING, false, false, generation)); + return store; + } + + private static MigrationOperationSnapshot verifying(MigrationOperationSnapshot current) { + return snapshot(current.applyMode(), MigrationOperationState.RUNNING, MigrationStage.VERIFYING, + 100, STARTED, VerificationState.RUNNING, false, false, + current.managedCandidateGeneration()); + } + + private static MigrationOperationSnapshot snapshot( + ApplyMode applyMode, + MigrationOperationState state, + MigrationStage stage, + int progress, + Instant startedAt, + VerificationState verification, + boolean activationAvailable, + boolean externalApplyRequired, + String generation) { + return new MigrationOperationSnapshot( + OPERATION, state, MigrationTarget.MYSQL, applyMode, stage, progress, + CREATED, startedAt, null, verification, null, null, + state == MigrationOperationState.RUNNING || state == MigrationOperationState.PENDING ? 1000 : 0, + activationAvailable, false, externalApplyRequired, IDENTITY, generation); + } + + private static RetainedCopyJournalContext context() { + return new RetainedCopyJournalContext(OPERATION, IDENTITY); + } + + private byte[] readJournal() { + try { + return Files.readAllBytes(root.resolve(FileMigrationOperationStore.RELATIVE_PATH)); + } catch (IOException failure) { + throw new AssertionError(failure); + } + } + + private static DurableRetainedCopyJournalHandoff handoff( + FileMigrationOperationStore store, ApplyMode applyMode) { + String generation = applyMode == ApplyMode.MANAGED_WRITE ? GENERATION : null; + DurableCutoverDraft draft = new DurableCutoverDraft( + OPERATION, MigrationTarget.MYSQL, applyMode, CREATED, STARTED, generation); + return new DurableRetainedCopyJournalHandoff(draft, store); + } + + private static void assertSafeFailure(SetupErrorCode code, Runnable action) { + assertThatThrownBy(action::run) + .isInstanceOfSatisfying(RetainedCopyJournalHandoffException.class, + failure -> assertThat(failure.errorCode()).isEqualTo(code)) + .hasNoCause() + .hasMessageNotContaining("private path") + .hasMessageNotContaining("jdbc:") + .hasMessageNotContaining("password") + .hasMessageNotContaining("username"); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinatorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinatorTest.java index 82f27e87a0..6a378b8b46 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinatorTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinatorTest.java @@ -31,6 +31,7 @@ import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceLease; import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceOrchestrator; import org.apache.hertzbeat.manager.maintenance.MigrationSourceAction; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; import org.apache.hertzbeat.manager.setup.config.SecretValue; import org.junit.jupiter.api.Test; @@ -59,7 +60,8 @@ class RetainedCutoverCoordinatorTest { assertThat(result.targetIdentityHash()).isEqualTo(IDENTITY); InOrder order = inOrder( fixture.factory, fixture.provisionLease, fixture.provisioner, - fixture.copyLease, fixture.maintenance, fixture.maintenanceLease, fixture.executor); + fixture.copyLease, fixture.maintenance, fixture.maintenanceLease, + fixture.executor, fixture.handoff); order.verify(fixture.factory).acquire(same(TARGET), same(fixture.password), anyDeadline()); order.verify(fixture.provisionLease).withConnection(any()); order.verify(fixture.provisioner).provision( @@ -74,6 +76,8 @@ class RetainedCutoverCoordinatorTest { eq(MetadataDatabaseKind.POSTGRESQL), anyDeadline(), same(MetadataMigrationProgressSink.NO_OP)); order.verify(fixture.copyLease).close(); + order.verify(fixture.handoff).handoff( + new RetainedCopyJournalContext(OPERATION_ID, IDENTITY)); verify(fixture.maintenanceLease, never()).close(); verify(fixture.factory, never()).close(); assertThat(fixture.provisionConnection).isNotSameAs(fixture.copyConnection); @@ -147,7 +151,8 @@ class RetainedCutoverCoordinatorTest { assertConflict(fixture::execute); assertConflict(() -> fixture.coordinator.execute( "operation-b", TARGET, fixture.password, TIMEOUT, - MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP)); + MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP, + RetainedCopyJournalHandoff.NO_OP)); } @Test @@ -207,14 +212,103 @@ class RetainedCutoverCoordinatorTest { assertThatThrownBy(fixture::execute) .isInstanceOf(RetainedCutoverReleaseRequiredException.class); + verifyNoInteractions(fixture.handoff); RetainedCutoverResult result = fixture.coordinator.retryRelease( OPERATION_ID, Duration.ofSeconds(1)); assertThat(result.status()).isEqualTo(RetainedCutoverResult.Status.RETAINED_SUCCESS); verify(fixture.executor).execute(any(), any(), any(), anyDeadline(), any()); + verify(fixture.handoff).handoff(new RetainedCopyJournalContext(OPERATION_ID, IDENTITY)); verify(fixture.maintenanceLease, never()).close(); } + @Test + void failedMandatoryHandoffRetainsFenceAndRetryUsesTheOriginallyBoundCallbackOnly() { + Fixture fixture = new Fixture(); + when(fixture.handoff.handoff(new RetainedCopyJournalContext(OPERATION_ID, IDENTITY))) + .thenThrow(new RetainedCopyJournalHandoffException( + SetupErrorCode.CONFIG_RECOVERY_REQUIRED)) + .thenReturn(RetainedCopyJournalDisposition.TRANSITIONED); + + assertThatThrownBy(fixture::execute) + .isInstanceOf(RetainedCopyJournalHandoffException.class) + .hasNoCause(); + assertConflict(() -> fixture.coordinator.retained(OPERATION_ID)); + assertConflict(() -> fixture.coordinator.releaseRetained(OPERATION_ID)); + assertConflict(() -> fixture.coordinator.retryHandoff("operation-b")); + + RetainedCutoverResult retried = fixture.coordinator.retryHandoff(OPERATION_ID); + + assertThat(retried.status()).isEqualTo(RetainedCutoverResult.Status.RETAINED_SUCCESS); + verify(fixture.handoff, times(2)).handoff( + new RetainedCopyJournalContext(OPERATION_ID, IDENTITY)); + verify(fixture.executor).execute(any(), any(), any(), anyDeadline(), any()); + verify(fixture.maintenanceLease, never()).close(); + } + + @Test + void privateHandoffRuntimeIsRedactedAndLeavesTheFencePending() { + Fixture fixture = new Fixture(); + when(fixture.handoff.handoff(new RetainedCopyJournalContext(OPERATION_ID, IDENTITY))) + .thenThrow(new IllegalStateException("private journal details")); + + assertThatThrownBy(fixture::execute) + .isInstanceOfSatisfying(RetainedCopyJournalHandoffException.class, failure -> + assertThat(failure.errorCode()).isEqualTo( + SetupErrorCode.CONFIG_RECOVERY_REQUIRED)) + .hasNoCause() + .hasMessageNotContaining("private journal details"); + assertConflict(() -> fixture.coordinator.retained(OPERATION_ID)); + assertConflict(() -> fixture.coordinator.releaseRetained(OPERATION_ID)); + verify(fixture.maintenanceLease, never()).close(); + } + + @Test + void handoffInterruptIsPreservedAndRetainsTheFence() { + Fixture fixture = new Fixture(); + when(fixture.handoff.handoff(new RetainedCopyJournalContext(OPERATION_ID, IDENTITY))) + .thenAnswer(invocation -> { + Thread.currentThread().interrupt(); + return RetainedCopyJournalDisposition.TRANSITIONED; + }); + + try { + assertThatThrownBy(fixture::execute) + .isInstanceOf(RetainedCopyJournalHandoffException.class) + .hasNoCause(); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + assertConflict(() -> fixture.coordinator.releaseRetained(OPERATION_ID)); + verify(fixture.maintenanceLease, never()).close(); + } finally { + Thread.interrupted(); + } + } + + @Test + void handoffFatalRemainsPrimaryAndRetainsTheFence() { + Fixture fixture = new Fixture(); + AssertionError fatal = new AssertionError("handoff fatal"); + when(fixture.handoff.handoff(new RetainedCopyJournalContext(OPERATION_ID, IDENTITY))) + .thenThrow(fatal); + + assertThatThrownBy(fixture::execute).isSameAs(fatal); + assertConflict(() -> fixture.coordinator.releaseRetained(OPERATION_ID)); + verify(fixture.maintenanceLease, never()).close(); + } + + @Test + void handoffReentryConflictsWhileTheBoundCallbackIsActive() { + Fixture fixture = new Fixture(); + when(fixture.handoff.handoff(new RetainedCopyJournalContext(OPERATION_ID, IDENTITY))) + .thenAnswer(invocation -> { + assertConflict(() -> fixture.coordinator.retryHandoff(OPERATION_ID)); + assertConflict(() -> fixture.coordinator.releaseRetained(OPERATION_ID)); + return RetainedCopyJournalDisposition.TRANSITIONED; + }); + + assertThat(fixture.execute().status()).isEqualTo(RetainedCutoverResult.Status.RETAINED_SUCCESS); + } + @Test void maintenanceReleaseFailureRetainsExactLeaseAndRetryDoesNotRepeatCopy() { Fixture fixture = new Fixture(); @@ -296,6 +390,7 @@ class RetainedCutoverCoordinatorTest { private final MigrationMaintenanceOrchestrator maintenance = mock(MigrationMaintenanceOrchestrator.class); private final MigrationMaintenanceLease maintenanceLease = mock(MigrationMaintenanceLease.class); private final JdbcMetadataMigrationExecutor executor = mock(JdbcMetadataMigrationExecutor.class); + private final RetainedCopyJournalHandoff handoff = mock(RetainedCopyJournalHandoff.class); private final SecretValue password = mock(SecretValue.class); private final AtomicLong ticker = new AtomicLong(); private final RetainedCutoverCoordinator coordinator; @@ -316,6 +411,7 @@ class RetainedCutoverCoordinatorTest { .thenReturn(new TargetSchemaProvisioningOutcome( TargetSchemaConnectionDisposition.REUSABLE)); when(maintenance.acquire(eq(OPERATION_ID), any())).thenReturn(maintenanceLease); + when(handoff.handoff(any())).thenReturn(RetainedCopyJournalDisposition.TRANSITIONED); coordinator = new RetainedCutoverCoordinator( factory, provisioner, maintenance, executor, ticker::get); } @@ -323,7 +419,7 @@ class RetainedCutoverCoordinatorTest { private RetainedCutoverResult execute() { return coordinator.execute( OPERATION_ID, TARGET, password, TIMEOUT, - MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP); + MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP, handoff); } private static void scopedTarget(TargetJdbcConnectionLease lease, Connection connection) { diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverFactoryRaceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverFactoryRaceTest.java index 4df56d54b7..8725c844c1 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverFactoryRaceTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverFactoryRaceTest.java @@ -120,7 +120,8 @@ class RetainedCutoverFactoryRaceTest { RetainedCutoverCoordinator coordinator, String operationId, SecretValue password) { return coordinator.execute( operationId, TARGET, password, TIMEOUT, - MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP); + MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP, + RetainedCopyJournalHandoff.NO_OP); } private static void assertConflict(Runnable action) { diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverFailureTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverFailureTest.java index 179cfbadd3..d6cff03186 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverFailureTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverFailureTest.java @@ -169,7 +169,8 @@ class RetainedCutoverFailureTest { fixture.password, Duration.ZERO, MetadataMigrationProgressSink.NO_OP, - RetainedCutoverPreparation.NO_OP)) + RetainedCutoverPreparation.NO_OP, + RetainedCopyJournalHandoff.NO_OP)) .isInstanceOf(MetadataMigrationException.class); assertThat(fixture.execute().status()).isEqualTo(RetainedCutoverResult.Status.RETAINED_SUCCESS); @@ -212,7 +213,8 @@ class RetainedCutoverFailureTest { .isInstanceOf(RetainedCutoverReleaseRequiredException.class); assertConflict(() -> fixture.coordinator.execute( "operation-b", TARGET, fixture.password, TIMEOUT, - MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP)); + MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP, + RetainedCopyJournalHandoff.NO_OP)); verify(fixture.factory).acquire(any(), any(), anyDeadline()); verifyNoInteractions(fixture.provisioner, fixture.maintenance, fixture.executor); @@ -269,7 +271,8 @@ class RetainedCutoverFailureTest { private RetainedCutoverResult execute() { return coordinator.execute( OPERATION_ID, TARGET, password, TIMEOUT, - MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP); + MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP, + RetainedCopyJournalHandoff.NO_OP); } private static void scopedTarget(TargetJdbcConnectionLease lease, Connection connection) { diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverJournalHandoffTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverJournalHandoffTest.java new file mode 100644 index 0000000000..f1be4a850f --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverJournalHandoffTest.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.nio.file.Path; +import java.sql.Connection; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceErrorCode; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceException; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceLease; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceOrchestrator; +import org.apache.hertzbeat.manager.maintenance.MigrationSourceAction; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.apache.hertzbeat.manager.setup.security.CommittedSetupFileDurabilityException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; + +@Timeout(15) +class RetainedCutoverJournalHandoffTest { + + private static final String OPERATION = "operation-a"; + private static final String IDENTITY = "a".repeat(64); + private static final Duration TIMEOUT = Duration.ofSeconds(1); + private static final MetadataDatabaseSettings TARGET = new MetadataDatabaseSettings( + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db.example/hertzbeat", "migration"); + + @TempDir + private Path root; + + @Test + void uncertainVerifyingConfirmationKeepsFenceAndHealthyRetryNeverRecopies() { + JournalFixture fixture = fixture((publication, target, content, publisher) -> { + publisher.publish(target, content); + if (publication <= 2) { + throw new CommittedSetupFileDurabilityException(); + } + }); + + assertPendingThenRetry(fixture, MigrationStage.VERIFYING, + RetainedCutoverResult.Status.RETAINED_SUCCESS); + } + + @Test + void uncertainFinalConfirmationKeepsFenceAndHealthyRetryReturnsAlreadyRetained() { + JournalFixture fixture = fixture((publication, target, content, publisher) -> { + publisher.publish(target, content); + if (publication == 2 || publication == 3) { + throw new CommittedSetupFileDurabilityException(); + } + }); + + assertPendingThenRetry(fixture, MigrationStage.READY_TO_ACTIVATE, + RetainedCutoverResult.Status.ALREADY_RETAINED); + } + + private void assertPendingThenRetry( + JournalFixture fixture, + MigrationStage durableStage, + RetainedCutoverResult.Status retryStatus) { + assertThatThrownBy(fixture::execute) + .isInstanceOf(RetainedCopyJournalHandoffException.class) + .hasNoCause(); + assertThat(fixture.store.find(OPERATION).orElseThrow().stage()).isEqualTo(durableStage); + assertConflict(() -> fixture.coordinator.releaseRetained(OPERATION)); + + fixture.delegate.set(new DurableRetainedCopyJournalHandoff(fixture.draft, fixture.store)); + assertThat(fixture.coordinator.retryHandoff(OPERATION).status()).isEqualTo(retryStatus); + verify(fixture.executor).execute(any(), any(), any(), anyDeadline(), any()); + verify(fixture.maintenanceLease, never()).close(); + } + + private JournalFixture fixture(PublicationBehavior behavior) { + DurableCutoverDraft draft = new DurableCutoverDraft( + OPERATION, MigrationTarget.POSTGRESQL, ApplyMode.MANAGED_WRITE, + Instant.parse("2026-08-10T03:00:00Z"), + Instant.parse("2026-08-10T03:00:01Z"), "candidate-generation"); + DurableCutoverSnapshots snapshots = new DurableCutoverSnapshots(draft, IDENTITY); + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + store.create(snapshots.cleanPending()); + store.compareAndTransition( + OPERATION, MigrationOperationState.PENDING, snapshots.running()); + MigrationOperationFilePublisher publisher = new MigrationOperationFilePublisher(root); + AtomicInteger publications = new AtomicInteger(); + FileMigrationOperationStore uncertain = new FileMigrationOperationStore(root, (target, content) -> + behavior.publish(publications.incrementAndGet(), target, content, publisher)); + return new JournalFixture(draft, store, uncertain); + } + + private static JdbcMetadataMigrationDeadline anyDeadline() { + return any(JdbcMetadataMigrationDeadline.class); + } + + private static void assertConflict(Runnable action) { + assertThatThrownBy(action::run) + .isInstanceOfSatisfying(MigrationMaintenanceException.class, failure -> + assertThat(failure.code()).isEqualTo( + MigrationMaintenanceErrorCode.MIGRATION_OPERATION_CONFLICT)); + } + + @FunctionalInterface + private interface PublicationBehavior { + void publish( + int publication, + Path target, + byte[] content, + MigrationOperationFilePublisher publisher) throws IOException; + } + + private static final class JournalFixture { + + private final DurableCutoverDraft draft; + private final FileMigrationOperationStore store; + private final AtomicReference delegate; + private final JdbcMetadataMigrationExecutor executor = mock(JdbcMetadataMigrationExecutor.class); + private final MigrationMaintenanceLease maintenanceLease = mock(MigrationMaintenanceLease.class); + private final SecretValue password = mock(SecretValue.class); + private final RetainedCopyJournalHandoff handoff; + private final RetainedCutoverCoordinator coordinator; + + private JournalFixture( + DurableCutoverDraft draft, + FileMigrationOperationStore store, + FileMigrationOperationStore uncertainStore) { + this.draft = draft; + this.store = store; + delegate = new AtomicReference<>(new DurableRetainedCopyJournalHandoff(draft, uncertainStore)); + TargetJdbcConnectionFactory factory = mock(TargetJdbcConnectionFactory.class); + TargetJdbcConnectionLease provisionLease = lease(IDENTITY); + TargetJdbcConnectionLease copyLease = lease(IDENTITY); + when(factory.acquire(same(TARGET), same(password), anyDeadline())) + .thenReturn(provisionLease, copyLease); + FlywayTargetSchemaProvisioner provisioner = mock(FlywayTargetSchemaProvisioner.class); + when(provisioner.provision(any(), any(), anyDeadline())).thenReturn( + new TargetSchemaProvisioningOutcome(TargetSchemaConnectionDisposition.REUSABLE)); + MigrationMaintenanceOrchestrator maintenance = mock(MigrationMaintenanceOrchestrator.class); + when(maintenance.acquire(eq(OPERATION), any())).thenReturn(maintenanceLease); + scopedSource(maintenanceLease, mock(Connection.class)); + RetainedCopyJournalHandoff handoff = context -> delegate.get().handoff(context); + coordinator = new RetainedCutoverCoordinator( + factory, provisioner, maintenance, executor, System::nanoTime); + this.handoff = handoff; + } + + private RetainedCutoverResult execute() { + return coordinator.execute( + OPERATION, TARGET, password, TIMEOUT, MetadataMigrationProgressSink.NO_OP, + RetainedCutoverPreparation.NO_OP, handoff); + } + + private static TargetJdbcConnectionLease lease(String identity) { + TargetJdbcConnectionLease lease = mock(TargetJdbcConnectionLease.class); + when(lease.targetIdentityHash()).thenReturn(identity); + Connection connection = mock(Connection.class); + doAnswer(invocation -> { + TargetJdbcConnectionAction action = invocation.getArgument(0); + action.execute(connection); + return null; + }).when(lease).withConnection(any()); + return lease; + } + + private static void scopedSource(MigrationMaintenanceLease lease, Connection connection) { + doAnswer(invocation -> { + MigrationSourceAction action = invocation.getArgument(0); + action.execute(connection); + return null; + }).when(lease).withSourceConnection(any()); + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverLifecycleTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverLifecycleTest.java index 127750976c..cb6d92d9b4 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverLifecycleTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverLifecycleTest.java @@ -63,7 +63,8 @@ class RetainedCutoverLifecycleTest { assertConflict(fixture::execute); assertConflict(() -> fixture.coordinator.execute( "operation-b", TARGET, fixture.password, TIMEOUT, - MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP)); + MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP, + RetainedCopyJournalHandoff.NO_OP)); } finally { releaseProvision.countDown(); first.get(2, TimeUnit.SECONDS); @@ -178,7 +179,8 @@ class RetainedCutoverLifecycleTest { private RetainedCutoverResult execute(SecretValue borrowedPassword) { return coordinator.execute( OPERATION_ID, TARGET, borrowedPassword, TIMEOUT, - MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP); + MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP, + RetainedCopyJournalHandoff.NO_OP); } private static void scopedTarget(TargetJdbcConnectionLease lease, Connection connection) { diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverPreparationTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverPreparationTest.java index 07dd7736e7..dad0b49d72 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverPreparationTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverPreparationTest.java @@ -213,7 +213,8 @@ class RetainedCutoverPreparationTest { capture(sameOperation, () -> fixture.execute(RetainedCutoverPreparation.NO_OP)); capture(foreignOperation, () -> fixture.coordinator.execute( "operation-b", TARGET, fixture.password, TIMEOUT, - MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP)); + MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP, + RetainedCopyJournalHandoff.NO_OP)); }; fixture.execute(preparation); @@ -290,7 +291,8 @@ class RetainedCutoverPreparationTest { private RetainedCutoverResult execute(RetainedCutoverPreparation preparation) { return coordinator.execute( OPERATION, TARGET, password, TIMEOUT, - MetadataMigrationProgressSink.NO_OP, preparation); + MetadataMigrationProgressSink.NO_OP, preparation, + RetainedCopyJournalHandoff.NO_OP); } private static void scopedTarget(TargetJdbcConnectionLease lease, Connection connection) { From f0e016beb132b7e8891e46eed45e4d6a75bb0cec Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 15:45:59 +0800 Subject: [PATCH 57/71] Activate retained metadata cutovers --- ...agedMigrationConfigurationTransaction.java | 29 ++- .../DurableRetainedManagedActivation.java | 122 +++++++++ .../workflow/RetainedCutoverCoordinator.java | 41 +++ .../setup/workflow/RetainedCutoverState.java | 77 +++++- .../workflow/RetainedManagedActivation.java | 15 ++ .../RetainedManagedActivationClaim.java | 35 +++ .../RetainedManagedActivationContext.java | 23 ++ .../RetainedManagedActivationDisposition.java | 14 ++ .../RetainedManagedActivationException.java | 27 ++ .../RetainedManagedActivationResult.java | 30 +++ .../RetainedManagedActivationSnapshots.java | 80 ++++++ .../ManagedMigrationActivationTest.java | 19 ++ .../DurableRetainedManagedActivationTest.java | 211 ++++++++++++++++ .../RetainedCutoverManagedActivationTest.java | 237 ++++++++++++++++++ 14 files changed, 951 insertions(+), 9 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableRetainedManagedActivation.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivation.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivationClaim.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivationContext.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivationDisposition.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivationException.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivationResult.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivationSnapshots.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DurableRetainedManagedActivationTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverManagedActivationTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransaction.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransaction.java index 44ac3bdf02..2163f7dcb7 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransaction.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransaction.java @@ -94,12 +94,15 @@ public final class ManagedMigrationConfigurationTransaction { /** Activates only the exact candidate over its recorded base generation. */ public ActivationOutcome activate(CandidateRef reference) throws IOException { Objects.requireNonNull(reference, "reference"); - return lock.execute(() -> store.withMaterial(reference, material -> { - if (material.inspection().state() != CandidateState.READY) { - return ActivationOutcome.RECOVERY_REQUIRED; - } - return activation.activate(material); - })); + return lock.execute(() -> activateMaterial(reference, null)); + } + + /** Activates only a ready candidate whose target identity exactly matches the journal. */ + public ActivationOutcome activateExact( + CandidateRef reference, String expectedTargetIdentityHash) throws IOException { + Objects.requireNonNull(reference, "reference"); + requireIdentityHash(expectedTargetIdentityHash); + return lock.execute(() -> activateMaterial(reference, expectedTargetIdentityHash)); } /** Restores only the exact recorded base while the candidate generation remains active. */ @@ -113,6 +116,20 @@ public final class ManagedMigrationConfigurationTransaction { })); } + private ActivationOutcome activateMaterial( + CandidateRef reference, String expectedTargetIdentityHash) throws IOException { + return store.withMaterial(reference, material -> { + Inspection inspection = material.inspection(); + if (inspection.state() != CandidateState.READY + || expectedTargetIdentityHash != null + && !inspection.targetIdentityHash().orElseThrow() + .equals(expectedTargetIdentityHash)) { + return ActivationOutcome.RECOVERY_REQUIRED; + } + return activation.activate(material); + }); + } + static void requireGeneration(String value, String label) { Objects.requireNonNull(value, label); if (!GENERATION.matcher(value).matches()) { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableRetainedManagedActivation.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableRetainedManagedActivation.java new file mode 100644 index 0000000000..114027f85f --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableRetainedManagedActivation.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.IOException; +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.ActivationOutcome; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.CandidateRef; + +/** Activates an exact managed candidate between two durable journal transitions. */ +final class DurableRetainedManagedActivation implements RetainedManagedActivation { + + private final DurableCutoverDraft draft; + private final FileMigrationOperationStore store; + private final ManagedMigrationConfigurationTransaction configuration; + + DurableRetainedManagedActivation( + DurableCutoverDraft draft, + FileMigrationOperationStore store, + ManagedMigrationConfigurationTransaction configuration) { + this.draft = Objects.requireNonNull(draft, "draft"); + this.store = Objects.requireNonNull(store, "store"); + this.configuration = Objects.requireNonNull(configuration, "configuration"); + if (draft.applyMode() != ApplyMode.MANAGED_WRITE) { + throw new IllegalArgumentException("Managed activation requires a managed cutover draft"); + } + } + + @Override + public RetainedManagedActivationDisposition activate( + RetainedManagedActivationContext context) { + Objects.requireNonNull(context, "context"); + try { + return activateDurably(context); + } catch (RetainedManagedActivationException failure) { + throw failure; + } catch (MigrationOperationStoreException failure) { + throw new RetainedManagedActivationException(failure.errorCode()); + } catch (IOException failure) { + throw recoveryRequired(); + } catch (RuntimeException failure) { + throw recoveryRequired(); + } + } + + private RetainedManagedActivationDisposition activateDurably( + RetainedManagedActivationContext context) throws IOException { + MigrationOperationSnapshot current = store.find(context.operationId()) + .orElseThrow(() -> new RetainedManagedActivationException( + SetupErrorCode.OPERATION_NOT_FOUND)); + requireExactIdentity(current, context); + RetainedManagedActivationSnapshots snapshots = + new RetainedManagedActivationSnapshots(current); + if (snapshots.awaitingRestart()) { + confirmExact(current); + return RetainedManagedActivationDisposition.ALREADY_AWAITING_RESTART; + } + MigrationOperationSnapshot activating; + if (snapshots.ready()) { + activating = snapshots.activatingSnapshot(); + store.compareAndTransitionOrConfirmDisposition( + context.operationId(), MigrationOperationState.READY_TO_ACTIVATE, activating); + } else if (snapshots.activating()) { + activating = current; + confirmExact(current); + } else { + throw new RetainedManagedActivationException(SetupErrorCode.OPERATION_CONFLICT); + } + ActivationOutcome outcome = activateCandidate(context); + if (outcome != ActivationOutcome.ACTIVATED && outcome != ActivationOutcome.ALREADY_ACTIVE) { + throw recoveryRequired(); + } + MigrationOperationSnapshot awaiting = + new RetainedManagedActivationSnapshots(activating).awaitingRestartSnapshot(); + store.compareAndTransitionOrConfirmDisposition( + context.operationId(), MigrationOperationState.RUNNING, awaiting); + return RetainedManagedActivationDisposition.ACTIVATED; + } + + private ActivationOutcome activateCandidate( + RetainedManagedActivationContext context) throws IOException { + CandidateRef candidate = new CandidateRef( + context.operationId(), draft.candidateGeneration()); + return configuration.activateExact(candidate, context.targetIdentityHash()); + } + + private void requireExactIdentity( + MigrationOperationSnapshot current, RetainedManagedActivationContext context) { + if (!draft.operationId().equals(context.operationId()) + || !current.operationId().equals(draft.operationId()) + || current.target() != draft.target() + || current.applyMode() != draft.applyMode() + || !current.createdAt().equals(draft.createdAt()) + || !Objects.equals(current.startedAt(), draft.startedAt()) + || !current.targetIdentityHash().equals(context.targetIdentityHash()) + || !Objects.equals(current.managedCandidateGeneration(), draft.candidateGeneration())) { + throw new RetainedManagedActivationException(SetupErrorCode.OPERATION_CONFLICT); + } + } + + private void confirmExact(MigrationOperationSnapshot current) { + FileMigrationOperationStore.ExactTransitionDisposition disposition = + store.compareAndTransitionOrConfirmDisposition( + current.operationId(), current.state(), current); + if (disposition != FileMigrationOperationStore.ExactTransitionDisposition.ALREADY_CONFIRMED) { + throw recoveryRequired(); + } + } + + private RetainedManagedActivationException recoveryRequired() { + return new RetainedManagedActivationException(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java index ce605a3f96..7bb3ee6326 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java @@ -118,6 +118,18 @@ final class RetainedCutoverCoordinator { return runHandoff(state.claimPendingHandoff(operationId)); } + RetainedManagedActivationResult activateRetained( + String operationId, RetainedManagedActivation activation) { + requireOperationId(operationId); + Objects.requireNonNull(activation, "activation"); + return runActivation(state.claimManagedActivation(operationId, activation)); + } + + RetainedManagedActivationResult retryActivation(String operationId) { + requireOperationId(operationId); + return runActivation(state.claimPendingActivation(operationId)); + } + private TargetJdbcConnectionLease acquire( RetainedCutoverState.Execution execution, MetadataDatabaseSettings target, @@ -230,6 +242,35 @@ final class RetainedCutoverCoordinator { } } + private RetainedManagedActivationResult runActivation( + RetainedManagedActivationClaim claim) { + if (claim.completed()) { + return claim.replay(); + } + RetainedCutoverState.Execution execution = claim.execution(); + try { + if (Thread.currentThread().isInterrupted()) { + throw new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT); + } + RetainedManagedActivationDisposition disposition = Objects.requireNonNull( + execution.activation().activate(execution.activationContext()), + "activation disposition"); + if (Thread.currentThread().isInterrupted()) { + throw new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT); + } + return state.completeActivation(execution, disposition); + } catch (Error fatal) { + state.activationPending(execution); + throw fatal; + } catch (RuntimeException failure) { + state.activationPending(execution); + if (failure instanceof RetainedManagedActivationException activationFailure) { + throw activationFailure; + } + throw new RetainedManagedActivationException(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + } + private void releasePending( RetainedCutoverState.Execution execution, RetainedCutoverRelease release) { state.releasePending(execution, release); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverState.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverState.java index c42ab677b3..bc2c033f7a 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverState.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverState.java @@ -50,6 +50,30 @@ final class RetainedCutoverState { return execution; } + synchronized RetainedManagedActivationClaim claimManagedActivation( + String operationId, RetainedManagedActivation activation) { + Execution execution = requireActive(operationId); + if (execution.phase == Phase.AWAITING_RESTART_RETAINED) { + return RetainedManagedActivationClaim.replay(execution.activationResult( + RetainedManagedActivationResult.Status.ALREADY_AWAITING_RESTART)); + } + requirePhase(execution, Phase.RETAINED); + execution.activation = Objects.requireNonNull(activation, "activation"); + execution.phase = Phase.ACTIVATING; + return RetainedManagedActivationClaim.execute(execution); + } + + synchronized RetainedManagedActivationClaim claimPendingActivation(String operationId) { + Execution execution = requireActive(operationId); + if (execution.phase == Phase.AWAITING_RESTART_RETAINED) { + return RetainedManagedActivationClaim.replay(execution.activationResult( + RetainedManagedActivationResult.Status.ALREADY_AWAITING_RESTART)); + } + requirePhase(execution, Phase.ACTIVATION_PENDING); + execution.phase = Phase.ACTIVATING; + return RetainedManagedActivationClaim.execute(execution); + } + synchronized void releasePending(Execution execution, RetainedCutoverRelease release) { if (active == execution) { execution.release = release; @@ -87,6 +111,27 @@ final class RetainedCutoverState { execution.phase = Phase.HANDOFF_PENDING; } + synchronized RetainedManagedActivationResult completeActivation( + Execution execution, RetainedManagedActivationDisposition disposition) { + if (active != execution || execution.phase != Phase.ACTIVATING) { + throw MigrationMaintenanceException.operationConflict(); + } + RetainedManagedActivationResult.Status status = + disposition == RetainedManagedActivationDisposition.ACTIVATED + ? RetainedManagedActivationResult.Status.ACTIVATED + : RetainedManagedActivationResult.Status.ALREADY_AWAITING_RESTART; + RetainedManagedActivationResult result = execution.activationResult(status); + execution.phase = Phase.AWAITING_RESTART_RETAINED; + return result; + } + + synchronized void activationPending(Execution execution) { + if (active != execution || execution.phase != Phase.ACTIVATING) { + throw MigrationMaintenanceException.operationConflict(); + } + execution.phase = Phase.ACTIVATION_PENDING; + } + synchronized void clear(Execution execution) { if (active == execution) { active = null; @@ -94,18 +139,29 @@ final class RetainedCutoverState { } private Execution require(String operationId, Phase phase) { - if (active == null - || active.phase != phase - || !active.operationId.equals(operationId)) { + Execution execution = requireActive(operationId); + requirePhase(execution, phase); + return execution; + } + + private Execution requireActive(String operationId) { + if (active == null || !active.operationId.equals(operationId)) { throw MigrationMaintenanceException.operationConflict(); } return active; } + private void requirePhase(Execution execution, Phase phase) { + if (execution.phase != phase) { + throw MigrationMaintenanceException.operationConflict(); + } + } + static final class Execution { private final String operationId; private final RetainedCopyJournalHandoff handoff; + private RetainedManagedActivation activation; private String targetIdentityHash; private MigrationMaintenanceLease maintenanceLease; private RetainedCutoverRelease release; @@ -132,9 +188,21 @@ final class RetainedCutoverState { return handoff; } + RetainedManagedActivationContext activationContext() { + return new RetainedManagedActivationContext(operationId, targetIdentityHash); + } + + RetainedManagedActivation activation() { + return activation; + } + RetainedCutoverResult result(RetainedCutoverResult.Status status) { return new RetainedCutoverResult(operationId, targetIdentityHash, status); } + + RetainedManagedActivationResult activationResult(RetainedManagedActivationResult.Status status) { + return new RetainedManagedActivationResult(operationId, targetIdentityHash, status); + } } private enum Phase { @@ -142,6 +210,9 @@ final class RetainedCutoverState { HANDOFFING, HANDOFF_PENDING, RETAINED, + ACTIVATING, + ACTIVATION_PENDING, + AWAITING_RESTART_RETAINED, RELEASING, RELEASE_PENDING } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivation.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivation.java new file mode 100644 index 0000000000..2acf0a8f03 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivation.java @@ -0,0 +1,15 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +/** Synchronous, secret-free activation performed while the exact maintenance fence is retained. */ +@FunctionalInterface +interface RetainedManagedActivation { + + RetainedManagedActivationDisposition activate(RetainedManagedActivationContext context); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivationClaim.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivationClaim.java new file mode 100644 index 0000000000..26335c6d9c --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivationClaim.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; + +/** Atomic state claim that either runs activation or replays its retained completion. */ +record RetainedManagedActivationClaim( + RetainedCutoverState.Execution execution, + RetainedManagedActivationResult replay) { + + RetainedManagedActivationClaim { + if ((execution == null) == (replay == null)) { + throw new IllegalArgumentException("An activation claim must execute or replay"); + } + } + + static RetainedManagedActivationClaim execute(RetainedCutoverState.Execution execution) { + return new RetainedManagedActivationClaim( + Objects.requireNonNull(execution, "execution"), null); + } + + static RetainedManagedActivationClaim replay(RetainedManagedActivationResult replay) { + return new RetainedManagedActivationClaim(null, Objects.requireNonNull(replay, "replay")); + } + + boolean completed() { + return replay != null; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivationContext.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivationContext.java new file mode 100644 index 0000000000..ebce32205c --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivationContext.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import org.apache.hertzbeat.manager.setup.api.OperationIdValidator; + +/** Secret-free exact identity of a retained managed cutover awaiting activation. */ +record RetainedManagedActivationContext(String operationId, String targetIdentityHash) { + + RetainedManagedActivationContext { + if (!OperationIdValidator.isSafe(operationId)) { + throw new IllegalArgumentException("Invalid operation id"); + } + if (targetIdentityHash == null || !targetIdentityHash.matches("[0-9a-f]{64}")) { + throw new IllegalArgumentException("Invalid target identity"); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivationDisposition.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivationDisposition.java new file mode 100644 index 0000000000..9a287a6ce1 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivationDisposition.java @@ -0,0 +1,14 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +/** Stable result of an exact managed activation and its journal transition. */ +enum RetainedManagedActivationDisposition { + ACTIVATED, + ALREADY_AWAITING_RESTART +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivationException.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivationException.java new file mode 100644 index 0000000000..5e463d6f18 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivationException.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Stable, cause-free activation failure that leaves the exact maintenance fence retained. */ +final class RetainedManagedActivationException extends RuntimeException { + + private final SetupErrorCode errorCode; + + RetainedManagedActivationException(SetupErrorCode errorCode) { + super("Retained managed activation failed: " + + Objects.requireNonNull(errorCode, "errorCode").value()); + this.errorCode = errorCode; + } + + SetupErrorCode errorCode() { + return errorCode; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivationResult.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivationResult.java new file mode 100644 index 0000000000..9fc696c2ce --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivationResult.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.OperationIdValidator; + +/** Secret-free result while restart ownership remains behind the retained fence. */ +record RetainedManagedActivationResult(String operationId, String targetIdentityHash, Status status) { + + RetainedManagedActivationResult { + if (!OperationIdValidator.isSafe(operationId)) { + throw new IllegalArgumentException("Invalid operation id"); + } + if (targetIdentityHash == null || !targetIdentityHash.matches("[0-9a-f]{64}")) { + throw new IllegalArgumentException("Invalid target identity"); + } + Objects.requireNonNull(status, "status"); + } + + enum Status { + ACTIVATED, + ALREADY_AWAITING_RESTART + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivationSnapshots.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivationSnapshots.java new file mode 100644 index 0000000000..1ac2cba8b2 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedManagedActivationSnapshots.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; + +/** Recognizes and builds exact journal shapes for retained managed activation. */ +final class RetainedManagedActivationSnapshots { + + private static final long ACTIVE_POLL_MILLIS = 1000; + + private final MigrationOperationSnapshot source; + + RetainedManagedActivationSnapshots(MigrationOperationSnapshot source) { + this.source = source; + } + + boolean ready() { + return source.applyMode() == ApplyMode.MANAGED_WRITE + && source.state() == MigrationOperationState.READY_TO_ACTIVATE + && source.stage() == MigrationStage.READY_TO_ACTIVATE + && exactCommon() + && source.activationAvailable() + && !source.restartRequired(); + } + + boolean activating() { + return source.applyMode() == ApplyMode.MANAGED_WRITE + && source.state() == MigrationOperationState.RUNNING + && source.stage() == MigrationStage.ACTIVATING + && exactCommon() + && !source.activationAvailable() + && !source.restartRequired(); + } + + boolean awaitingRestart() { + return source.applyMode() == ApplyMode.MANAGED_WRITE + && source.state() == MigrationOperationState.AWAITING_RESTART + && source.stage() == MigrationStage.AWAITING_RESTART + && exactCommon() + && !source.activationAvailable() + && source.restartRequired(); + } + + MigrationOperationSnapshot activatingSnapshot() { + return snapshot(MigrationOperationState.RUNNING, MigrationStage.ACTIVATING, false); + } + + MigrationOperationSnapshot awaitingRestartSnapshot() { + return snapshot(MigrationOperationState.AWAITING_RESTART, + MigrationStage.AWAITING_RESTART, true); + } + + private boolean exactCommon() { + return source.progressPercent() == 100 + && source.startedAt() != null + && source.completedAt() == null + && source.verificationState() == VerificationState.SUCCEEDED + && source.errorCode() == null + && source.rollbackOrigin() == null + && !source.externalApplyRequired(); + } + + private MigrationOperationSnapshot snapshot( + MigrationOperationState state, MigrationStage stage, boolean restartRequired) { + return new MigrationOperationSnapshot( + source.operationId(), state, source.target(), source.applyMode(), stage, 100, + source.createdAt(), source.startedAt(), null, VerificationState.SUCCEEDED, + null, null, ACTIVE_POLL_MILLIS, false, restartRequired, false, + source.targetIdentityHash(), source.managedCandidateGeneration()); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationActivationTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationActivationTest.java index 2a71c6a837..e7a0586347 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationActivationTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationActivationTest.java @@ -67,6 +67,25 @@ class ManagedMigrationActivationTest { migration.rollback(reference)); } + @Test + void exactActivationRejectsJournalIdentityMismatchWithoutChangingActiveConfiguration() + throws Exception { + ManagedConfigurationTransaction setup = new ManagedConfigurationTransaction(installationRoot); + assertEquals(ManagedConfigurationTransaction.Outcome.APPLIED, setup.apply(bundle("base"))); + String baseGeneration = activeGeneration(installationRoot); + ManagedMigrationConfigurationTransaction migration = + new ManagedMigrationConfigurationTransaction(installationRoot); + ManagedMigrationConfigurationTransaction.CandidateRef reference = + migration.stage(OPERATION, CANDIDATE, baseGeneration, IDENTITY, bundle("next")); + + assertEquals(ManagedMigrationConfigurationTransaction.ActivationOutcome.RECOVERY_REQUIRED, + migration.activateExact(reference, "f".repeat(64))); + + assertEquals(baseGeneration, activeGeneration(installationRoot)); + assertEquals(ManagedMigrationConfigurationTransaction.CandidateState.READY, + migration.inspect(reference).state()); + } + @Test void laterActiveGenerationMakesActivationAndRollbackStaleWithoutWriting() throws Exception { ManagedConfigurationTransaction setup = new ManagedConfigurationTransaction(installationRoot); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DurableRetainedManagedActivationTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DurableRetainedManagedActivationTest.java new file mode 100644 index 0000000000..d8c4e8028f --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DurableRetainedManagedActivationTest.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.nio.file.Path; +import java.time.Instant; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.ActivationOutcome; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.CandidateRef; +import org.apache.hertzbeat.manager.setup.security.CommittedSetupFileDurabilityException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +class DurableRetainedManagedActivationTest { + + private static final String OPERATION = "operation-a"; + private static final String IDENTITY = "a".repeat(64); + private static final String GENERATION = "candidate-generation"; + private static final Instant CREATED = Instant.parse("2026-08-10T03:00:00Z"); + private static final Instant STARTED = CREATED.plusSeconds(1); + private static final CandidateRef CANDIDATE = new CandidateRef(OPERATION, GENERATION); + + @TempDir + private Path root; + + private DurableCutoverDraft draft; + private FileMigrationOperationStore store; + private ManagedMigrationConfigurationTransaction configuration; + + @BeforeEach + void setUp() { + draft = new DurableCutoverDraft(OPERATION, MigrationTarget.POSTGRESQL, + ApplyMode.MANAGED_WRITE, CREATED, STARTED, GENERATION); + store = new FileMigrationOperationStore(root); + DurableCutoverSnapshots preparation = new DurableCutoverSnapshots(draft, IDENTITY); + store.create(preparation.cleanPending()); + store.compareAndTransition(OPERATION, MigrationOperationState.PENDING, preparation.running()); + new DurableRetainedCopyJournalHandoff(draft, store) + .handoff(new RetainedCopyJournalContext(OPERATION, IDENTITY)); + configuration = mock(ManagedMigrationConfigurationTransaction.class); + } + + @Test + void persistsActivatingBeforeExactConfigAndAwaitingRestartAfterIt() throws Exception { + when(configuration.activateExact(CANDIDATE, IDENTITY)).thenAnswer(invocation -> { + assertThat(store.find(OPERATION).orElseThrow().stage()).isEqualTo(MigrationStage.ACTIVATING); + return ActivationOutcome.ACTIVATED; + }); + DurableRetainedManagedActivation activation = + new DurableRetainedManagedActivation(draft, store, configuration); + + assertThat(activation.activate(new RetainedManagedActivationContext(OPERATION, IDENTITY))) + .isEqualTo(RetainedManagedActivationDisposition.ACTIVATED); + + MigrationOperationSnapshot current = store.find(OPERATION).orElseThrow(); + assertThat(current.state()).isEqualTo(MigrationOperationState.AWAITING_RESTART); + assertThat(current.stage()).isEqualTo(MigrationStage.AWAITING_RESTART); + verify(configuration).activateExact(CANDIDATE, IDENTITY); + } + + @ParameterizedTest + @EnumSource(value = ActivationOutcome.class, names = {"RECOVERY_REQUIRED", "STALE"}) + void rejectedConfigOutcomeLeavesDurableActivatingForSameOperationRetry( + ActivationOutcome rejected) throws Exception { + when(configuration.activateExact(CANDIDATE, IDENTITY)) + .thenReturn(rejected) + .thenReturn(ActivationOutcome.ALREADY_ACTIVE); + DurableRetainedManagedActivation activation = + new DurableRetainedManagedActivation(draft, store, configuration); + + assertThatThrownBy(() -> activation.activate( + new RetainedManagedActivationContext(OPERATION, IDENTITY))) + .isInstanceOfSatisfying(RetainedManagedActivationException.class, failure -> + assertThat(failure.errorCode()).isEqualTo(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)) + .hasNoCause(); + assertThat(store.find(OPERATION).orElseThrow().stage()).isEqualTo(MigrationStage.ACTIVATING); + + assertThat(activation.activate(new RetainedManagedActivationContext(OPERATION, IDENTITY))) + .isEqualTo(RetainedManagedActivationDisposition.ACTIVATED); + verify(configuration, org.mockito.Mockito.times(2)).activateExact(CANDIDATE, IDENTITY); + } + + @Test + void alreadyAwaitingRestartConfirmsExactConfigWithoutJournalRegression() throws Exception { + when(configuration.activateExact(CANDIDATE, IDENTITY)).thenReturn(ActivationOutcome.ACTIVATED); + DurableRetainedManagedActivation first = + new DurableRetainedManagedActivation(draft, store, configuration); + first.activate(new RetainedManagedActivationContext(OPERATION, IDENTITY)); + when(configuration.activateExact(CANDIDATE, IDENTITY)).thenReturn(ActivationOutcome.ALREADY_ACTIVE); + DurableRetainedManagedActivation replay = + new DurableRetainedManagedActivation(draft, store, configuration); + + assertThat(replay.activate(new RetainedManagedActivationContext(OPERATION, IDENTITY))) + .isEqualTo(RetainedManagedActivationDisposition.ALREADY_AWAITING_RESTART); + assertThat(store.find(OPERATION).orElseThrow().stage()).isEqualTo(MigrationStage.AWAITING_RESTART); + verify(configuration, org.mockito.Mockito.times(1)).activateExact(CANDIDATE, IDENTITY); + verify(configuration, never()).rollback(CANDIDATE); + } + + @Test + void activatingJournalFailureStopsBeforeConfigurationAndRetryConverges() throws Exception { + FileMigrationOperationStore failingStore = failPublication(1); + when(configuration.activateExact(CANDIDATE, IDENTITY)) + .thenReturn(ActivationOutcome.ACTIVATED); + DurableRetainedManagedActivation activation = + new DurableRetainedManagedActivation(draft, failingStore, configuration); + + assertRecovery(() -> activation.activate( + new RetainedManagedActivationContext(OPERATION, IDENTITY))); + assertThat(store.find(OPERATION).orElseThrow().stage()) + .isEqualTo(MigrationStage.READY_TO_ACTIVATE); + verify(configuration, never()).activateExact(CANDIDATE, IDENTITY); + + assertThat(activation.activate(new RetainedManagedActivationContext(OPERATION, IDENTITY))) + .isEqualTo(RetainedManagedActivationDisposition.ACTIVATED); + } + + @Test + void finalJournalFailureLeavesActivatingAndRetryDoesNotRepeatConfigMutation() throws Exception { + FileMigrationOperationStore failingStore = failPublication(2); + when(configuration.activateExact(CANDIDATE, IDENTITY)) + .thenReturn(ActivationOutcome.ACTIVATED) + .thenReturn(ActivationOutcome.ALREADY_ACTIVE); + DurableRetainedManagedActivation activation = + new DurableRetainedManagedActivation(draft, failingStore, configuration); + + assertRecovery(() -> activation.activate( + new RetainedManagedActivationContext(OPERATION, IDENTITY))); + assertThat(store.find(OPERATION).orElseThrow().stage()).isEqualTo(MigrationStage.ACTIVATING); + + assertThat(activation.activate(new RetainedManagedActivationContext(OPERATION, IDENTITY))) + .isEqualTo(RetainedManagedActivationDisposition.ACTIVATED); + verify(configuration, org.mockito.Mockito.times(2)).activateExact(CANDIDATE, IDENTITY); + } + + @Test + void committedFinalJournalWithFailedConfirmationRetriesWithoutReactivatingConfig() + throws Exception { + MigrationOperationFilePublisher delegate = new MigrationOperationFilePublisher(root); + AtomicInteger publications = new AtomicInteger(); + FileMigrationOperationStore uncertainStore = new FileMigrationOperationStore( + root, (target, content) -> { + delegate.publish(target, content); + int publication = publications.incrementAndGet(); + if (publication == 2 || publication == 3) { + throw new CommittedSetupFileDurabilityException(); + } + }); + when(configuration.activateExact(CANDIDATE, IDENTITY)) + .thenReturn(ActivationOutcome.ACTIVATED); + DurableRetainedManagedActivation activation = + new DurableRetainedManagedActivation(draft, uncertainStore, configuration); + + assertRecovery(() -> activation.activate( + new RetainedManagedActivationContext(OPERATION, IDENTITY))); + assertThat(store.find(OPERATION).orElseThrow().stage()) + .isEqualTo(MigrationStage.AWAITING_RESTART); + + assertThat(activation.activate(new RetainedManagedActivationContext(OPERATION, IDENTITY))) + .isEqualTo(RetainedManagedActivationDisposition.ALREADY_AWAITING_RESTART); + verify(configuration, org.mockito.Mockito.times(1)).activateExact(CANDIDATE, IDENTITY); + } + + private FileMigrationOperationStore failPublication(int failureIndex) { + MigrationOperationFilePublisher delegate = new MigrationOperationFilePublisher(root); + AtomicInteger publications = new AtomicInteger(); + return new FileMigrationOperationStore(root, (target, content) -> { + if (publications.incrementAndGet() == failureIndex) { + throw new IOException("simulated journal publication failure"); + } + delegate.publish(target, content); + }); + } + + private static void assertRecovery(ThrowingAction action) { + assertThatThrownBy(action::run) + .isInstanceOfSatisfying(RetainedManagedActivationException.class, failure -> + assertThat(failure.errorCode()).isIn( + SetupErrorCode.CONFIG_WRITE_FAILED, + SetupErrorCode.CONFIG_RECOVERY_REQUIRED)) + .hasNoCause() + .hasMessageNotContaining("simulated"); + } + + @FunctionalInterface + private interface ThrowingAction { + void run() throws Exception; + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverManagedActivationTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverManagedActivationTest.java new file mode 100644 index 0000000000..a45566bbc2 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverManagedActivationTest.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceException; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceLease; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceOrchestrator; +import org.apache.hertzbeat.manager.maintenance.MigrationSourceAction; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +@Timeout(10) +class RetainedCutoverManagedActivationTest { + + private static final String OPERATION = "operation-a"; + private static final String IDENTITY = "a".repeat(64); + private static final MetadataDatabaseSettings TARGET = new MetadataDatabaseSettings( + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db.example/hertzbeat", "migration"); + private static final Duration TIMEOUT = Duration.ofSeconds(1); + + @Test + void claimsTheFenceBeforeActivationAndKeepsItForRestartHandoff() { + Fixture fixture = new Fixture(); + fixture.execute(); + RetainedManagedActivation activation = context -> { + assertThat(context).isEqualTo(new RetainedManagedActivationContext(OPERATION, IDENTITY)); + assertConflict(() -> fixture.coordinator.releaseRetained(OPERATION)); + assertConflict(() -> fixture.coordinator.activateRetained(OPERATION, ignored -> + RetainedManagedActivationDisposition.ACTIVATED)); + assertConflict(() -> fixture.coordinator.retryActivation(OPERATION)); + return RetainedManagedActivationDisposition.ACTIVATED; + }; + + RetainedManagedActivationResult result = fixture.coordinator.activateRetained(OPERATION, activation); + + assertThat(result.status()).isEqualTo(RetainedManagedActivationResult.Status.ACTIVATED); + assertThat(result.operationId()).isEqualTo(OPERATION); + assertThat(result.targetIdentityHash()).isEqualTo(IDENTITY); + assertConflict(() -> fixture.coordinator.releaseRetained(OPERATION)); + assertConflict(() -> fixture.coordinator.retained(OPERATION)); + verify(fixture.maintenanceLease, never()).close(); + } + + @Test + void sameOperationRetryUsesTheBoundActivationWithoutRecopying() { + Fixture fixture = new Fixture(); + fixture.execute(); + RetainedManagedActivation activation = mock(RetainedManagedActivation.class); + when(activation.activate(any())) + .thenThrow(new RetainedManagedActivationException( + org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode + .CONFIG_RECOVERY_REQUIRED)) + .thenReturn(RetainedManagedActivationDisposition.ALREADY_AWAITING_RESTART); + + assertThatThrownBy(() -> fixture.coordinator.activateRetained(OPERATION, activation)) + .isInstanceOf(RetainedManagedActivationException.class) + .hasNoCause(); + assertConflict(() -> fixture.coordinator.releaseRetained(OPERATION)); + + RetainedManagedActivationResult result = fixture.coordinator.retryActivation(OPERATION); + + assertThat(result.status()).isEqualTo( + RetainedManagedActivationResult.Status.ALREADY_AWAITING_RESTART); + verify(activation, times(2)).activate(new RetainedManagedActivationContext(OPERATION, IDENTITY)); + verify(fixture.executor).execute(any(), any(), any(), anyDeadline(), any()); + verify(fixture.maintenanceLease, never()).close(); + } + + @Test + void lostSuccessResponseReplaysAwaitingRestartWithoutCallingAnyActivationAgain() { + Fixture fixture = new Fixture(); + fixture.execute(); + RetainedManagedActivation original = mock(RetainedManagedActivation.class); + RetainedManagedActivation replacement = mock(RetainedManagedActivation.class); + when(original.activate(any())).thenReturn(RetainedManagedActivationDisposition.ACTIVATED); + + assertThat(fixture.coordinator.activateRetained(OPERATION, original).status()) + .isEqualTo(RetainedManagedActivationResult.Status.ACTIVATED); + assertThat(fixture.coordinator.activateRetained(OPERATION, replacement).status()) + .isEqualTo(RetainedManagedActivationResult.Status.ALREADY_AWAITING_RESTART); + assertThat(fixture.coordinator.retryActivation(OPERATION).status()) + .isEqualTo(RetainedManagedActivationResult.Status.ALREADY_AWAITING_RESTART); + + verify(original, times(1)).activate(any()); + verify(replacement, never()).activate(any()); + verify(fixture.executor).execute(any(), any(), any(), anyDeadline(), any()); + verify(fixture.maintenanceLease, never()).close(); + } + + @Test + void privateRuntimeIsRedactedAndRetainsTheExactFenceForRetry() { + Fixture fixture = new Fixture(); + fixture.execute(); + RetainedManagedActivation activation = mock(RetainedManagedActivation.class); + when(activation.activate(any())) + .thenThrow(new IllegalStateException("private-path")) + .thenReturn(RetainedManagedActivationDisposition.ACTIVATED); + + assertThatThrownBy(() -> fixture.coordinator.activateRetained(OPERATION, activation)) + .isInstanceOfSatisfying(RetainedManagedActivationException.class, failure -> + assertThat(failure.errorCode()).isEqualTo(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)) + .hasNoCause() + .hasMessageNotContaining("private-path"); + assertConflict(() -> fixture.coordinator.retryActivation("operation-b")); + + assertThat(fixture.coordinator.retryActivation(OPERATION).status()) + .isEqualTo(RetainedManagedActivationResult.Status.ACTIVATED); + verify(fixture.executor).execute(any(), any(), any(), anyDeadline(), any()); + verify(fixture.maintenanceLease, never()).close(); + } + + @Test + void fatalRemainsPrimaryAndRetainsTheExactFenceForRetry() { + Fixture fixture = new Fixture(); + fixture.execute(); + AssertionError fatal = new AssertionError("activation fatal"); + RetainedManagedActivation activation = mock(RetainedManagedActivation.class); + when(activation.activate(any())) + .thenThrow(fatal) + .thenReturn(RetainedManagedActivationDisposition.ACTIVATED); + + assertThatThrownBy(() -> fixture.coordinator.activateRetained(OPERATION, activation)) + .isSameAs(fatal); + + assertThat(fixture.coordinator.retryActivation(OPERATION).status()) + .isEqualTo(RetainedManagedActivationResult.Status.ACTIVATED); + verify(fixture.maintenanceLease, never()).close(); + } + + @Test + void interruptStopsBeforeCallbackAndIsPreservedForExplicitRetry() { + Fixture fixture = new Fixture(); + fixture.execute(); + RetainedManagedActivation activation = mock(RetainedManagedActivation.class); + when(activation.activate(any())).thenReturn(RetainedManagedActivationDisposition.ACTIVATED); + + Thread.currentThread().interrupt(); + try { + assertThatThrownBy(() -> fixture.coordinator.activateRetained(OPERATION, activation)) + .isInstanceOfSatisfying(RetainedManagedActivationException.class, failure -> + assertThat(failure.errorCode()) + .isEqualTo(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)) + .hasNoCause(); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + verify(activation, never()).activate(any()); + } finally { + Thread.interrupted(); + } + + assertThat(fixture.coordinator.retryActivation(OPERATION).status()) + .isEqualTo(RetainedManagedActivationResult.Status.ACTIVATED); + verify(fixture.maintenanceLease, never()).close(); + } + + private static JdbcMetadataMigrationDeadline anyDeadline() { + return any(JdbcMetadataMigrationDeadline.class); + } + + private static void assertConflict(Runnable action) { + assertThatThrownBy(action::run).isInstanceOf(MigrationMaintenanceException.class); + } + + private static final class Fixture { + + private final Connection targetConnection = mock(Connection.class); + private final Connection sourceConnection = mock(Connection.class); + private final TargetJdbcConnectionFactory factory = mock(TargetJdbcConnectionFactory.class); + private final TargetJdbcConnectionLease provisionLease = mock(TargetJdbcConnectionLease.class); + private final TargetJdbcConnectionLease copyLease = mock(TargetJdbcConnectionLease.class); + private final FlywayTargetSchemaProvisioner provisioner = mock(FlywayTargetSchemaProvisioner.class); + private final MigrationMaintenanceOrchestrator maintenance = mock(MigrationMaintenanceOrchestrator.class); + private final MigrationMaintenanceLease maintenanceLease = mock(MigrationMaintenanceLease.class); + private final JdbcMetadataMigrationExecutor executor = mock(JdbcMetadataMigrationExecutor.class); + private final SecretValue password = mock(SecretValue.class); + private final RetainedCutoverCoordinator coordinator; + + private Fixture() { + when(provisionLease.targetIdentityHash()).thenReturn(IDENTITY); + when(copyLease.targetIdentityHash()).thenReturn(IDENTITY); + when(factory.acquire(same(TARGET), same(password), anyDeadline())) + .thenReturn(provisionLease, copyLease); + scopedTarget(provisionLease); + scopedTarget(copyLease); + scopedSource(maintenanceLease); + when(provisioner.provision(any(), any(), anyDeadline())).thenReturn( + new TargetSchemaProvisioningOutcome(TargetSchemaConnectionDisposition.REUSABLE)); + when(maintenance.acquire(eq(OPERATION), any())).thenReturn(maintenanceLease); + coordinator = new RetainedCutoverCoordinator( + factory, provisioner, maintenance, executor, new AtomicLong()::get); + } + + private void execute() { + coordinator.execute(OPERATION, TARGET, password, TIMEOUT, MetadataMigrationProgressSink.NO_OP, + RetainedCutoverPreparation.NO_OP, context -> RetainedCopyJournalDisposition.TRANSITIONED); + } + + private void scopedTarget(TargetJdbcConnectionLease lease) { + doAnswer(invocation -> { + TargetJdbcConnectionAction action = invocation.getArgument(0); + action.execute(targetConnection); + return null; + }).when(lease).withConnection(any()); + } + + private void scopedSource(MigrationMaintenanceLease lease) { + doAnswer(invocation -> { + MigrationSourceAction action = invocation.getArgument(0); + action.execute(sourceConnection); + return null; + }).when(lease).withSourceConnection(any()); + } + } +} From ffa2adbc91612f69d0cf63499f03fd877d916fa2 Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 16:08:09 +0800 Subject: [PATCH 58/71] Reconcile managed metadata startup --- ...agedMigrationConfigurationTransaction.java | 39 ++- .../setup/config/MigrationCandidateStore.java | 17 ++ .../workflow/FileMigrationOperationStore.java | 15 + .../ManagedMigrationStartupReconciler.java | 171 +++++++++++ .../MigrationStartupReconciliation.java | 18 ++ ...grationStartupReconciliationException.java | 26 ++ .../workflow/MigrationStartupSnapshots.java | 56 ++++ .../MigrationStartupTargetVerification.java | 15 + .../MigrationStartupTargetVerifier.java | 17 ++ .../ManagedMigrationExactAccessTest.java | 89 ++++++ ...ManagedMigrationStartupReconcilerTest.java | 286 ++++++++++++++++++ 11 files changed, 743 insertions(+), 6 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupReconciler.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupReconciliation.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupReconciliationException.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupSnapshots.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupTargetVerification.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupTargetVerifier.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationExactAccessTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupReconcilerTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransaction.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransaction.java index 2163f7dcb7..bc277bd3e2 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransaction.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransaction.java @@ -85,6 +85,16 @@ public final class ManagedMigrationConfigurationTransaction { return lock.execute(() -> store.readExact(reference, reader)); } + /** Reads only the ready candidate whose target identity exactly matches the journal. */ + public T readExact( + CandidateRef reference, String expectedTargetIdentityHash, CandidateReader reader) + throws IOException { + Objects.requireNonNull(reference, "reference"); + requireIdentityHash(expectedTargetIdentityHash); + Objects.requireNonNull(reader, "reader"); + return lock.execute(() -> store.readExact(reference, expectedTargetIdentityHash, reader)); + } + /** Removes only the operation-and-generation scoped candidate named by the reference. */ public DiscardOutcome discardExact(CandidateRef reference) throws IOException { Objects.requireNonNull(reference, "reference"); @@ -108,12 +118,15 @@ public final class ManagedMigrationConfigurationTransaction { /** Restores only the exact recorded base while the candidate generation remains active. */ public RollbackOutcome rollback(CandidateRef reference) throws IOException { Objects.requireNonNull(reference, "reference"); - return lock.execute(() -> store.withMaterial(reference, material -> { - if (material.inspection().state() != CandidateState.READY) { - return RollbackOutcome.RECOVERY_REQUIRED; - } - return activation.rollback(material); - })); + return lock.execute(() -> rollbackMaterial(reference, null)); + } + + /** Restores the exact base only when candidate and journal target identities match. */ + public RollbackOutcome rollbackExact( + CandidateRef reference, String expectedTargetIdentityHash) throws IOException { + Objects.requireNonNull(reference, "reference"); + requireIdentityHash(expectedTargetIdentityHash); + return lock.execute(() -> rollbackMaterial(reference, expectedTargetIdentityHash)); } private ActivationOutcome activateMaterial( @@ -130,6 +143,20 @@ public final class ManagedMigrationConfigurationTransaction { }); } + private RollbackOutcome rollbackMaterial( + CandidateRef reference, String expectedTargetIdentityHash) { + return store.withMaterial(reference, material -> { + Inspection inspection = material.inspection(); + if (inspection.state() != CandidateState.READY + || expectedTargetIdentityHash != null + && !inspection.targetIdentityHash().orElseThrow() + .equals(expectedTargetIdentityHash)) { + return RollbackOutcome.RECOVERY_REQUIRED; + } + return activation.rollback(material); + }); + } + static void requireGeneration(String value, String label) { Objects.requireNonNull(value, label); if (!GENERATION.matcher(value).matches()) { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateStore.java index 4ec0c5a58c..21b8836442 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateStore.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/MigrationCandidateStore.java @@ -133,6 +133,23 @@ final class MigrationCandidateStore { } } + T readExact( + ManagedMigrationConfigurationTransaction.CandidateRef reference, + String expectedTargetIdentityHash, + ManagedMigrationConfigurationTransaction.CandidateReader reader) throws IOException { + try (MigrationCandidateMaterial material = read(reference)) { + ManagedMigrationConfigurationTransaction.Inspection inspection = material.inspection(); + if (inspection.state() != ManagedMigrationConfigurationTransaction.CandidateState.READY + || !inspection.targetIdentityHash().orElseThrow() + .equals(expectedTargetIdentityHash)) { + throw new IOException("Managed migration candidate identity does not match"); + } + ManagedConfigurationBundle bundle = new ManagedConfigurationBundle( + material.application().orElseThrow(), material.secrets().orElseThrow()); + return reader.read(bundle); + } + } + T withMaterial(ManagedMigrationConfigurationTransaction.CandidateRef reference, MaterialReader reader) { try (MigrationCandidateMaterial material = read(reference)) { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java index 10e77c550b..a2b63e6449 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java @@ -100,6 +100,21 @@ public final class FileMigrationOperationStore implements MigrationOperationStor .filter(snapshot -> snapshot.operationId().equals(operationId)).findFirst()); } + /** Selects one startup record only when no other operation still owns migration progress. */ + Optional selectForStartup(String operationId) { + requireSafeId(operationId); + return locked(() -> { + List snapshots = read(); + if (snapshots.stream().anyMatch(snapshot -> !snapshot.terminal() + && !snapshot.operationId().equals(operationId))) { + throw failure(SetupErrorCode.OPERATION_CONFLICT); + } + return snapshots.stream() + .filter(snapshot -> snapshot.operationId().equals(operationId)) + .findFirst(); + }); + } + @Override public List history() { return locked(() -> List.copyOf(read())); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupReconciler.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupReconciler.java new file mode 100644 index 0000000000..5a10119c5c --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupReconciler.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.IOException; +import java.time.Clock; +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.ActivationOutcome; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.CandidateRef; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.RollbackOutcome; + +/** Reconciles one managed restart journal without Spring, JPA, or the current datasource. */ +final class ManagedMigrationStartupReconciler { + + private final DurableCutoverDraft draft; + private final FileMigrationOperationStore store; + private final ManagedMigrationConfigurationTransaction configuration; + private final MigrationStartupTargetVerifier verifier; + private final Clock clock; + + ManagedMigrationStartupReconciler( + DurableCutoverDraft draft, + FileMigrationOperationStore store, + ManagedMigrationConfigurationTransaction configuration, + MigrationStartupTargetVerifier verifier, + Clock clock) { + this.draft = Objects.requireNonNull(draft, "draft"); + this.store = Objects.requireNonNull(store, "store"); + this.configuration = Objects.requireNonNull(configuration, "configuration"); + this.verifier = Objects.requireNonNull(verifier, "verifier"); + this.clock = Objects.requireNonNull(clock, "clock"); + if (draft.applyMode() != ApplyMode.MANAGED_WRITE) { + throw new IllegalArgumentException("Startup reconciliation requires a managed migration"); + } + } + + MigrationStartupReconciliation reconcile() { + try { + return reconcileSafely(); + } catch (MigrationStartupReconciliationException failure) { + throw failure; + } catch (MigrationOperationStoreException failure) { + throw failure(failure.errorCode()); + } catch (IOException failure) { + throw failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } catch (RuntimeException failure) { + throw failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + } + + private MigrationStartupReconciliation reconcileSafely() throws IOException { + MigrationOperationSnapshot current = store.selectForStartup(draft.operationId()).orElse(null); + if (current == null) { + return MigrationStartupReconciliation.NO_MIGRATION; + } + requireExactDraft(current); + if (current.state() == MigrationOperationState.SUCCEEDED) { + confirm(current); + return MigrationStartupReconciliation.ALREADY_SUCCEEDED; + } + if (current.state() == MigrationOperationState.ROLLED_BACK) { + confirm(current); + return MigrationStartupReconciliation.ALREADY_ROLLED_BACK_RESTART_REQUIRED; + } + current = convergeActivation(current); + if (isRestartRollback(current)) { + return rollback(current); + } + if (!new RetainedManagedActivationSnapshots(current).awaitingRestart()) { + throw failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + CandidateRef candidate = candidate(); + MigrationStartupTargetVerification verification = + Objects.requireNonNull(verifier.verify(candidate, current.targetIdentityHash()), + "target verification"); + return switch (verification) { + case CONFIRMED -> succeed(current); + case DETERMINISTIC_MISMATCH -> beginRollback(current); + case TRANSIENT_UNAVAILABLE -> MigrationStartupReconciliation.GATED; + }; + } + + private MigrationOperationSnapshot convergeActivation(MigrationOperationSnapshot current) + throws IOException { + RetainedManagedActivationSnapshots snapshots = new RetainedManagedActivationSnapshots(current); + if (!snapshots.activating()) { + return current; + } + ActivationOutcome outcome = configuration.activateExact(candidate(), current.targetIdentityHash()); + if (outcome != ActivationOutcome.ACTIVATED && outcome != ActivationOutcome.ALREADY_ACTIVE) { + throw failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + MigrationOperationSnapshot awaiting = snapshots.awaitingRestartSnapshot(); + store.compareAndTransitionOrConfirmDisposition( + current.operationId(), MigrationOperationState.RUNNING, awaiting); + return awaiting; + } + + private MigrationStartupReconciliation succeed(MigrationOperationSnapshot current) { + MigrationOperationSnapshot succeeded = + new MigrationStartupSnapshots(current).succeeded(clock.instant()); + store.compareAndTransitionOrConfirmDisposition( + current.operationId(), MigrationOperationState.AWAITING_RESTART, succeeded); + return MigrationStartupReconciliation.SUCCEEDED; + } + + private MigrationStartupReconciliation beginRollback(MigrationOperationSnapshot current) + throws IOException { + MigrationOperationSnapshot rollingBack = new MigrationStartupSnapshots(current).rollingBack(); + store.compareAndTransitionOrConfirmDisposition( + current.operationId(), MigrationOperationState.AWAITING_RESTART, rollingBack); + return rollback(rollingBack); + } + + private MigrationStartupReconciliation rollback(MigrationOperationSnapshot current) + throws IOException { + RollbackOutcome outcome = configuration.rollbackExact(candidate(), current.targetIdentityHash()); + if (outcome != RollbackOutcome.ROLLED_BACK && outcome != RollbackOutcome.ALREADY_ROLLED_BACK) { + throw failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + MigrationOperationSnapshot rolledBack = + new MigrationStartupSnapshots(current).rolledBack(clock.instant()); + store.compareAndTransitionOrConfirmDisposition( + current.operationId(), MigrationOperationState.RUNNING, rolledBack); + return MigrationStartupReconciliation.ROLLED_BACK_RESTART_REQUIRED; + } + + private boolean isRestartRollback(MigrationOperationSnapshot current) { + return current.state() == MigrationOperationState.RUNNING + && current.stage() == MigrationStage.ROLLING_BACK + && current.rollbackOrigin() == MigrationRollbackOrigin.RESTART_FAILURE; + } + + private void requireExactDraft(MigrationOperationSnapshot current) { + if (!current.operationId().equals(draft.operationId()) + || current.target() != draft.target() + || current.applyMode() != draft.applyMode() + || !current.createdAt().equals(draft.createdAt()) + || !Objects.equals(current.startedAt(), draft.startedAt()) + || !Objects.equals(current.managedCandidateGeneration(), draft.candidateGeneration())) { + throw failure(SetupErrorCode.OPERATION_CONFLICT); + } + } + + private void confirm(MigrationOperationSnapshot current) { + FileMigrationOperationStore.ExactTransitionDisposition disposition = + store.compareAndTransitionOrConfirmDisposition( + current.operationId(), current.state(), current); + if (disposition != FileMigrationOperationStore.ExactTransitionDisposition.ALREADY_CONFIRMED) { + throw failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + } + + private CandidateRef candidate() { + return new CandidateRef(draft.operationId(), draft.candidateGeneration()); + } + + private MigrationStartupReconciliationException failure(SetupErrorCode code) { + return new MigrationStartupReconciliationException(code); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupReconciliation.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupReconciliation.java new file mode 100644 index 0000000000..a49e3f4f29 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupReconciliation.java @@ -0,0 +1,18 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +/** Secret-free startup reconciliation outcome. */ +enum MigrationStartupReconciliation { + NO_MIGRATION, + GATED, + SUCCEEDED, + ALREADY_SUCCEEDED, + ROLLED_BACK_RESTART_REQUIRED, + ALREADY_ROLLED_BACK_RESTART_REQUIRED +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupReconciliationException.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupReconciliationException.java new file mode 100644 index 0000000000..56dfcc51b7 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupReconciliationException.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Cause-free startup migration failure that keeps the business runtime gated. */ +final class MigrationStartupReconciliationException extends RuntimeException { + + private final SetupErrorCode errorCode; + + MigrationStartupReconciliationException(SetupErrorCode errorCode) { + super("Metadata migration startup reconciliation requires recovery"); + this.errorCode = Objects.requireNonNull(errorCode, "errorCode"); + } + + SetupErrorCode errorCode() { + return errorCode; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupSnapshots.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupSnapshots.java new file mode 100644 index 0000000000..d1a771e857 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupSnapshots.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.time.Instant; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Builds exact restart success and rollback journal snapshots. */ +final class MigrationStartupSnapshots { + + private static final long ACTIVE_POLL_MILLIS = 1000; + + private final MigrationOperationSnapshot source; + + MigrationStartupSnapshots(MigrationOperationSnapshot source) { + this.source = source; + } + + MigrationOperationSnapshot succeeded(Instant completedAt) { + return snapshot(MigrationOperationState.SUCCEEDED, MigrationStage.COMPLETED, + completedAt, null, null, 0); + } + + MigrationOperationSnapshot rollingBack() { + return snapshot(MigrationOperationState.RUNNING, MigrationStage.ROLLING_BACK, + null, null, MigrationRollbackOrigin.RESTART_FAILURE, ACTIVE_POLL_MILLIS); + } + + MigrationOperationSnapshot rolledBack(Instant completedAt) { + return snapshot(MigrationOperationState.ROLLED_BACK, MigrationStage.ROLLED_BACK, + completedAt, SetupErrorCode.RESTART_FAILED, + MigrationRollbackOrigin.RESTART_FAILURE, 0); + } + + private MigrationOperationSnapshot snapshot( + MigrationOperationState state, + MigrationStage stage, + Instant completedAt, + SetupErrorCode errorCode, + MigrationRollbackOrigin rollbackOrigin, + long nextPollAfterMillis) { + return new MigrationOperationSnapshot( + source.operationId(), state, source.target(), source.applyMode(), stage, 100, + source.createdAt(), source.startedAt(), completedAt, + source.verificationState(), errorCode, rollbackOrigin, nextPollAfterMillis, + false, false, false, source.targetIdentityHash(), + source.managedCandidateGeneration()); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupTargetVerification.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupTargetVerification.java new file mode 100644 index 0000000000..bc75c0176a --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupTargetVerification.java @@ -0,0 +1,15 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +/** Stable, secret-free result of read-only restart target verification. */ +enum MigrationStartupTargetVerification { + CONFIRMED, + DETERMINISTIC_MISMATCH, + TRANSIENT_UNAVAILABLE +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupTargetVerifier.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupTargetVerifier.java new file mode 100644 index 0000000000..2a4c81008f --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupTargetVerifier.java @@ -0,0 +1,17 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.CandidateRef; + +/** Read-only target verification boundary for startup reconciliation. */ +@FunctionalInterface +interface MigrationStartupTargetVerifier { + + MigrationStartupTargetVerification verify(CandidateRef candidate, String targetIdentityHash); +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationExactAccessTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationExactAccessTest.java new file mode 100644 index 0000000000..e78e07c682 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationExactAccessTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class ManagedMigrationExactAccessTest { + + private static final String OPERATION = "operation-a"; + private static final String GENERATION = "candidate-generation"; + private static final String IDENTITY = "a".repeat(64); + + @TempDir + private Path root; + + @Test + void identityBoundReadExposesOnlyTheExactCandidateAndKeepsSecretsOwned() throws Exception { + ManagedMigrationConfigurationTransaction migration = stagedAndActivated(); + ManagedMigrationConfigurationTransaction.CandidateRef candidate = candidate(); + + String url = migration.readExact(candidate, IDENTITY, + bundle -> bundle.application().metadataDatabase().jdbcUrl()); + + assertThat(url).isEqualTo("jdbc:postgresql://db.example/hertzbeat"); + } + + @Test + void identityMismatchDoesNotInvokeReaderOrRollbackActiveGeneration() throws Exception { + ManagedMigrationConfigurationTransaction migration = stagedAndActivated(); + ManagedMigrationConfigurationTransaction.CandidateRef candidate = candidate(); + AtomicBoolean invoked = new AtomicBoolean(); + + assertThatThrownBy(() -> migration.readExact(candidate, "f".repeat(64), bundle -> { + invoked.set(true); + return bundle.application(); + })).isInstanceOf(java.io.IOException.class).hasNoCause(); + assertThat(invoked).isFalse(); + assertThat(migration.rollbackExact(candidate, "f".repeat(64))) + .isEqualTo(ManagedMigrationConfigurationTransaction.RollbackOutcome.RECOVERY_REQUIRED); + assertThat(activeGeneration()).isEqualTo(GENERATION); + } + + private ManagedMigrationConfigurationTransaction stagedAndActivated() throws Exception { + ManagedConfigurationTransaction setup = new ManagedConfigurationTransaction(root); + assertThat(setup.apply(bundle(MetadataDatabaseKind.H2, "jdbc:h2:file:./data/hertzbeat", "sa"))) + .isEqualTo(ManagedConfigurationTransaction.Outcome.APPLIED); + String base = activeGeneration(); + ManagedMigrationConfigurationTransaction migration = + new ManagedMigrationConfigurationTransaction(root); + migration.stage(OPERATION, GENERATION, base, IDENTITY, + bundle(MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://db.example/hertzbeat", "hertzbeat")); + assertThat(migration.activateExact(candidate(), IDENTITY)) + .isEqualTo(ManagedMigrationConfigurationTransaction.ActivationOutcome.ACTIVATED); + return migration; + } + + private ManagedConfigurationBundle bundle( + MetadataDatabaseKind kind, String url, String username) { + ManagedApplicationConfig application = new ManagedApplicationConfig( + new MetadataDatabaseSettings(kind, url, username), + GreptimeSettings.anonymous( + new GreptimeEndpoints("greptime.example:4001", "http://greptime.example:4000"), + "public")); + return new ManagedConfigurationBundle(application, + ManagedSecrets.withoutTelemetryPassword(SecretValue.of("database-password"))); + } + + private ManagedMigrationConfigurationTransaction.CandidateRef candidate() { + return new ManagedMigrationConfigurationTransaction.CandidateRef(OPERATION, GENERATION); + } + + private String activeGeneration() { + return new FileManagedApplicationConfigStore(root) + .readActive().generation().orElseThrow(); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupReconcilerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupReconcilerTest.java new file mode 100644 index 0000000000..1f41d07714 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupReconcilerTest.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.nio.file.Path; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.ActivationOutcome; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.CandidateRef; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.RollbackOutcome; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class ManagedMigrationStartupReconcilerTest { + + private static final String OPERATION = "operation-a"; + private static final String IDENTITY = "a".repeat(64); + private static final String GENERATION = "candidate-generation"; + private static final Instant CREATED = Instant.parse("2026-08-10T03:00:00Z"); + private static final Instant STARTED = CREATED.plusSeconds(1); + private static final Instant COMPLETED = CREATED.plusSeconds(20); + private static final CandidateRef CANDIDATE = new CandidateRef(OPERATION, GENERATION); + + @TempDir + private Path root; + + private DurableCutoverDraft draft; + private FileMigrationOperationStore store; + private ManagedMigrationConfigurationTransaction configuration; + private MigrationStartupTargetVerifier verifier; + + @BeforeEach + void setUp() { + draft = new DurableCutoverDraft(OPERATION, MigrationTarget.POSTGRESQL, + ApplyMode.MANAGED_WRITE, CREATED, STARTED, GENERATION); + store = new FileMigrationOperationStore(root); + configuration = mock(ManagedMigrationConfigurationTransaction.class); + verifier = mock(MigrationStartupTargetVerifier.class); + seedReady(); + } + + @Test + void convergesActivatingBeforeReadOnlyTargetVerificationAndSuccessJournal() throws Exception { + MigrationOperationSnapshot ready = current(); + store.compareAndTransition(OPERATION, MigrationOperationState.READY_TO_ACTIVATE, + new RetainedManagedActivationSnapshots(ready).activatingSnapshot()); + when(configuration.activateExact(CANDIDATE, IDENTITY)).thenAnswer(invocation -> { + assertThat(current().stage()).isEqualTo(MigrationStage.ACTIVATING); + return ActivationOutcome.ALREADY_ACTIVE; + }); + when(verifier.verify(CANDIDATE, IDENTITY)).thenAnswer(invocation -> { + assertThat(current().state()).isEqualTo(MigrationOperationState.AWAITING_RESTART); + return MigrationStartupTargetVerification.CONFIRMED; + }); + + assertThat(reconciler().reconcile()) + .isEqualTo(MigrationStartupReconciliation.SUCCEEDED); + + assertThat(current().state()).isEqualTo(MigrationOperationState.SUCCEEDED); + assertThat(current().completedAt()).isEqualTo(COMPLETED); + verify(configuration).activateExact(CANDIDATE, IDENTITY); + verify(verifier).verify(CANDIDATE, IDENTITY); + } + + @Test + void awaitingRestartTargetConfirmationTransitionsDirectlyToSuccess() throws Exception { + seedAwaitingRestart(); + when(verifier.verify(CANDIDATE, IDENTITY)) + .thenReturn(MigrationStartupTargetVerification.CONFIRMED); + + assertThat(reconciler().reconcile()) + .isEqualTo(MigrationStartupReconciliation.SUCCEEDED); + + assertThat(current().state()).isEqualTo(MigrationOperationState.SUCCEEDED); + verify(configuration, never()).activateExact(CANDIDATE, IDENTITY); + } + + @Test + void deterministicMismatchRollsBackExactGenerationBeforeTerminalJournal() throws Exception { + seedAwaitingRestart(); + when(verifier.verify(CANDIDATE, IDENTITY)) + .thenReturn(MigrationStartupTargetVerification.DETERMINISTIC_MISMATCH); + when(configuration.rollbackExact(CANDIDATE, IDENTITY)).thenAnswer(invocation -> { + assertThat(current().stage()).isEqualTo(MigrationStage.ROLLING_BACK); + return RollbackOutcome.ROLLED_BACK; + }); + + assertThat(reconciler().reconcile()) + .isEqualTo(MigrationStartupReconciliation.ROLLED_BACK_RESTART_REQUIRED); + + MigrationOperationSnapshot terminal = current(); + assertThat(terminal.state()).isEqualTo(MigrationOperationState.ROLLED_BACK); + assertThat(terminal.errorCode()).isEqualTo(SetupErrorCode.RESTART_FAILED); + assertThat(terminal.rollbackOrigin()).isEqualTo(MigrationRollbackOrigin.RESTART_FAILURE); + verify(configuration).rollbackExact(CANDIDATE, IDENTITY); + } + + @Test + void transientTargetFailureKeepsAwaitingRestartGated() throws Exception { + seedAwaitingRestart(); + when(verifier.verify(CANDIDATE, IDENTITY)) + .thenReturn(MigrationStartupTargetVerification.TRANSIENT_UNAVAILABLE); + + assertThat(reconciler().reconcile()) + .isEqualTo(MigrationStartupReconciliation.GATED); + + assertThat(current().state()).isEqualTo(MigrationOperationState.AWAITING_RESTART); + verify(configuration, never()).rollbackExact(CANDIDATE, IDENTITY); + } + + @Test + void privateRuntimeFailureIsCauseFreeAndLeavesAwaitingRestartGated() throws Exception { + seedAwaitingRestart(); + when(verifier.verify(CANDIDATE, IDENTITY)) + .thenThrow(new IllegalStateException("private jdbc endpoint")); + + assertThatThrownBy(() -> reconciler().reconcile()) + .isInstanceOfSatisfying(MigrationStartupReconciliationException.class, failure -> + assertThat(failure.errorCode()) + .isEqualTo(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)) + .hasNoCause() + .hasMessageNotContaining("private") + .hasMessageNotContaining("jdbc") + .hasMessageNotContaining("endpoint"); + assertThat(current().state()).isEqualTo(MigrationOperationState.AWAITING_RESTART); + verify(configuration, never()).rollbackExact(CANDIDATE, IDENTITY); + } + + @Test + void terminalSuccessReplayDoesNotVerifyOrMutateAgain() throws Exception { + seedAwaitingRestart(); + when(verifier.verify(CANDIDATE, IDENTITY)) + .thenReturn(MigrationStartupTargetVerification.CONFIRMED); + ManagedMigrationStartupReconciler reconciler = reconciler(); + assertThat(reconciler.reconcile()).isEqualTo(MigrationStartupReconciliation.SUCCEEDED); + + assertThat(reconciler.reconcile()) + .isEqualTo(MigrationStartupReconciliation.ALREADY_SUCCEEDED); + + verify(verifier, times(1)).verify(CANDIDATE, IDENTITY); + verify(configuration, never()).rollbackExact(CANDIDATE, IDENTITY); + } + + @Test + void activationRecoveryRequiredKeepsActivatingJournalGated() throws Exception { + MigrationOperationSnapshot ready = current(); + store.compareAndTransition(OPERATION, MigrationOperationState.READY_TO_ACTIVATE, + new RetainedManagedActivationSnapshots(ready).activatingSnapshot()); + when(configuration.activateExact(CANDIDATE, IDENTITY)) + .thenReturn(ActivationOutcome.RECOVERY_REQUIRED); + + assertThatThrownBy(() -> reconciler().reconcile()) + .isInstanceOfSatisfying(MigrationStartupReconciliationException.class, failure -> + assertThat(failure.errorCode()) + .isEqualTo(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)); + + assertThat(current().stage()).isEqualTo(MigrationStage.ACTIVATING); + verify(verifier, never()).verify(CANDIDATE, IDENTITY); + } + + @Test + void rollbackRecoveryRequiredReplaysOnlyExactRollback() throws Exception { + seedAwaitingRestart(); + when(verifier.verify(CANDIDATE, IDENTITY)) + .thenReturn(MigrationStartupTargetVerification.DETERMINISTIC_MISMATCH); + when(configuration.rollbackExact(CANDIDATE, IDENTITY)) + .thenReturn(RollbackOutcome.RECOVERY_REQUIRED, RollbackOutcome.ALREADY_ROLLED_BACK); + ManagedMigrationStartupReconciler reconciler = reconciler(); + + assertThatThrownBy(reconciler::reconcile) + .isInstanceOfSatisfying(MigrationStartupReconciliationException.class, failure -> + assertThat(failure.errorCode()) + .isEqualTo(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)); + assertThat(current().stage()).isEqualTo(MigrationStage.ROLLING_BACK); + + assertThat(reconciler.reconcile()) + .isEqualTo(MigrationStartupReconciliation.ROLLED_BACK_RESTART_REQUIRED); + assertThat(reconciler.reconcile()) + .isEqualTo(MigrationStartupReconciliation.ALREADY_ROLLED_BACK_RESTART_REQUIRED); + verify(verifier, times(1)).verify(CANDIDATE, IDENTITY); + verify(configuration, times(2)).rollbackExact(CANDIDATE, IDENTITY); + } + + @Test + void targetVerifierErrorRemainsPrimaryAndDoesNotChangeJournal() { + seedAwaitingRestart(); + AssertionError fatal = new AssertionError("private target failure"); + when(verifier.verify(CANDIDATE, IDENTITY)).thenThrow(fatal); + + assertThatThrownBy(() -> reconciler().reconcile()).isSameAs(fatal); + + assertThat(current().state()).isEqualTo(MigrationOperationState.AWAITING_RESTART); + } + + @Test + void terminalDraftCannotHideForeignActiveOperation() throws Exception { + seedAwaitingRestart(); + when(verifier.verify(CANDIDATE, IDENTITY)) + .thenReturn(MigrationStartupTargetVerification.CONFIRMED); + ManagedMigrationStartupReconciler reconciler = reconciler(); + assertThat(reconciler.reconcile()).isEqualTo(MigrationStartupReconciliation.SUCCEEDED); + store.create(foreignPending()); + clearInvocations(configuration, verifier); + + assertThatThrownBy(reconciler::reconcile) + .isInstanceOfSatisfying(MigrationStartupReconciliationException.class, failure -> + assertThat(failure.errorCode()).isEqualTo(SetupErrorCode.OPERATION_CONFLICT)); + + verifyNoInteractions(configuration, verifier); + } + + @Test + void missingDraftCannotHideForeignActiveOperation() { + FileMigrationOperationStore foreignStore = new FileMigrationOperationStore(root.resolve("foreign")); + foreignStore.create(foreignPending()); + ManagedMigrationStartupReconciler reconciler = new ManagedMigrationStartupReconciler( + draft, foreignStore, configuration, verifier, + Clock.fixed(COMPLETED, ZoneOffset.UTC)); + + assertThatThrownBy(reconciler::reconcile) + .isInstanceOfSatisfying(MigrationStartupReconciliationException.class, failure -> + assertThat(failure.errorCode()).isEqualTo(SetupErrorCode.OPERATION_CONFLICT)); + + verifyNoInteractions(configuration, verifier); + } + + private ManagedMigrationStartupReconciler reconciler() { + return new ManagedMigrationStartupReconciler( + draft, store, configuration, verifier, + Clock.fixed(COMPLETED, ZoneOffset.UTC)); + } + + private void seedReady() { + DurableCutoverSnapshots preparation = new DurableCutoverSnapshots(draft, IDENTITY); + store.create(preparation.cleanPending()); + store.compareAndTransition(OPERATION, MigrationOperationState.PENDING, preparation.running()); + new DurableRetainedCopyJournalHandoff(draft, store) + .handoff(new RetainedCopyJournalContext(OPERATION, IDENTITY)); + } + + private void seedAwaitingRestart() { + if (current().state() == MigrationOperationState.AWAITING_RESTART) { + return; + } + MigrationOperationSnapshot ready = current(); + MigrationOperationSnapshot activating = + new RetainedManagedActivationSnapshots(ready).activatingSnapshot(); + store.compareAndTransition(OPERATION, MigrationOperationState.READY_TO_ACTIVATE, activating); + store.compareAndTransition(OPERATION, MigrationOperationState.RUNNING, + new RetainedManagedActivationSnapshots(activating).awaitingRestartSnapshot()); + } + + private MigrationOperationSnapshot current() { + return store.find(OPERATION).orElseThrow(); + } + + private MigrationOperationSnapshot foreignPending() { + DurableCutoverDraft foreign = new DurableCutoverDraft( + "operation-b", MigrationTarget.MYSQL, ApplyMode.MANAGED_WRITE, + CREATED.plusSeconds(30), STARTED.plusSeconds(30), "foreign-generation"); + return new DurableCutoverSnapshots(foreign, "b".repeat(64)).cleanPending(); + } +} From 65a8026baae8fe9bb682c4cb353b6fb79621ccf7 Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 16:38:51 +0800 Subject: [PATCH 59/71] Verify managed migration startup target --- ...MigrationStartupCurrentSchemaVerifier.java | 37 ++ ...eBackedMigrationStartupTargetVerifier.java | 227 +++++++++ ...MigrationStartupCurrentSchemaVerifier.java | 22 + .../MigrationStartupTargetInspector.java | 90 ++++ ...grationStartupTargetVerificationState.java | 165 ++++++ ...kedMigrationStartupTargetVerifierTest.java | 476 ++++++++++++++++++ .../MigrationStartupTargetInspectorTest.java | 121 +++++ 7 files changed, 1138 insertions(+) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/B206MigrationStartupCurrentSchemaVerifier.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/CandidateBackedMigrationStartupTargetVerifier.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupCurrentSchemaVerifier.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupTargetInspector.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupTargetVerificationState.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/CandidateBackedMigrationStartupTargetVerifierTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupTargetInspectorTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/B206MigrationStartupCurrentSchemaVerifier.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/B206MigrationStartupCurrentSchemaVerifier.java new file mode 100644 index 0000000000..1e1bfe1264 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/B206MigrationStartupCurrentSchemaVerifier.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Compares the target to the packaged B206 history and semantic schema contract without writes. */ +final class B206MigrationStartupCurrentSchemaVerifier implements MigrationStartupCurrentSchemaVerifier { + + @Override + public boolean isCurrent( + Connection connection, + MetadataDatabaseKind kind, + JdbcMetadataMigrationDeadline deadline) throws SQLException { + TargetSchemaJdbcBudget budget = new TargetSchemaJdbcBudget(deadline); + budget.check(); + TargetSchemaBaseline baseline; + try { + baseline = TargetSchemaBaseline.load(kind); + } catch (MetadataMigrationException timeout) { + throw timeout; + } catch (IOException | RuntimeException failure) { + throw new MigrationStartupReconciliationException(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + budget.check(); + return new FlywaySchemaHistory(kind).isCurrent(connection, baseline, budget); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/CandidateBackedMigrationStartupTargetVerifier.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/CandidateBackedMigrationStartupTargetVerifier.java new file mode 100644 index 0000000000..e91b116da5 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/CandidateBackedMigrationStartupTargetVerifier.java @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.IOException; +import java.time.Duration; +import java.util.Objects; +import java.util.Optional; +import java.util.function.LongSupplier; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigurationBundle; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.CandidateRef; +import org.apache.hertzbeat.manager.setup.installation.InstallationFingerprint; +import org.apache.hertzbeat.manager.setup.installation.LocalInstallationFingerprintStore; +import org.apache.hertzbeat.manager.setup.workflow.MigrationStartupTargetVerificationState.StableCompletion; +import org.apache.hertzbeat.manager.setup.workflow.MigrationStartupTargetVerificationState.VerificationContext; +import org.apache.hertzbeat.manager.setup.workflow.MigrationStartupTargetVerificationState.VerificationHolder; + +/** Verifies one exact managed target while candidate secrets remain synchronously borrowed. */ +final class CandidateBackedMigrationStartupTargetVerifier + implements MigrationStartupTargetVerifier, AutoCloseable { + + private final ManagedMigrationConfigurationTransaction configuration; + private final LocalInstallationFingerprintStore fingerprints; + private final TargetJdbcConnectionFactory factory; + private final MigrationStartupTargetInspector inspector; + private final Duration timeout; + private final LongSupplier ticker; + private final MigrationStartupTargetVerificationState state = + new MigrationStartupTargetVerificationState(); + private boolean closed; + + CandidateBackedMigrationStartupTargetVerifier( + ManagedMigrationConfigurationTransaction configuration, + LocalInstallationFingerprintStore fingerprints, + TargetJdbcConnectionFactory factory, + MigrationStartupTargetInspector inspector, + Duration timeout, + LongSupplier ticker) { + this.configuration = Objects.requireNonNull(configuration, "configuration"); + this.fingerprints = Objects.requireNonNull(fingerprints, "fingerprints"); + this.factory = Objects.requireNonNull(factory, "factory"); + this.inspector = Objects.requireNonNull(inspector, "inspector"); + this.timeout = Objects.requireNonNull(timeout, "timeout"); + this.ticker = Objects.requireNonNull(ticker, "ticker"); + } + + @Override + public synchronized MigrationStartupTargetVerification verify( + CandidateRef candidate, String targetIdentityHash) { + VerificationContext context = state.context(candidate, targetIdentityHash); + try { + if (closed) { + throw recovery(); + } + JdbcMetadataMigrationDeadline deadline = JdbcMetadataMigrationDeadline.start(timeout, ticker); + check(deadline); + if (state.hasPendingLease()) { + return state.closePending(context); + } + settleAcquire(context, deadline); + InstallationFingerprint fingerprint = readFingerprint(deadline); + return configuration.readExact(candidate, targetIdentityHash, + bundle -> verifyBundle(context, bundle, fingerprint, deadline)); + } catch (IOException failure) { + throw recovery(); + } catch (MigrationStartupReconciliationException failure) { + throw failure; + } catch (TargetJdbcConnectionException failure) { + return classifyAcquire(context, failure); + } catch (MetadataMigrationException timeoutFailure) { + return MigrationStartupTargetVerification.TRANSIENT_UNAVAILABLE; + } catch (RuntimeException failure) { + throw recovery(); + } + } + + @Override + public synchronized void close() { + if (closed) { + return; + } + StableCompletion pendingCompletion = null; + if (state.hasPendingLease()) { + pendingCompletion = state.closePendingForShutdown(); + } + try { + factory.close(); + } catch (RuntimeException failure) { + closed = true; + if (pendingCompletion != null && pendingCompletion.fatal() != null) { + throw pendingCompletion.fatal(); + } + throw recovery(); + } catch (Error fatal) { + closed = true; + if (pendingCompletion != null && pendingCompletion.fatal() != null) { + throw pendingCompletion.fatal(); + } + throw fatal; + } + closed = true; + replayPendingCompletion(pendingCompletion); + } + + private static void replayPendingCompletion(StableCompletion pendingCompletion) { + if (pendingCompletion != null) { + pendingCompletion.replay(); + } + } + + private MigrationStartupTargetVerification verifyBundle( + VerificationContext context, + ManagedConfigurationBundle bundle, + InstallationFingerprint fingerprint, + JdbcMetadataMigrationDeadline deadline) { + TargetJdbcConnectionLease lease = factory.acquire( + bundle.application().metadataDatabase(), + bundle.secrets().metadataDatabasePassword(), deadline); + StableCompletion completion = inspect(context, bundle, fingerprint, lease, deadline); + try { + lease.close(); + } catch (RuntimeException failure) { + state.retainLease(context, lease, completion); + if (completion.fatal() != null) { + throw completion.fatal(); + } + throw recovery(); + } catch (Error fatal) { + StableCompletion retained = completion.withFatalUnlessPresent(fatal); + state.retainLease(context, lease, retained); + throw retained.fatal(); + } + return completion.replay(); + } + + private StableCompletion inspect( + VerificationContext context, + ManagedConfigurationBundle bundle, + InstallationFingerprint fingerprint, + TargetJdbcConnectionLease lease, + JdbcMetadataMigrationDeadline deadline) { + try { + if (!lease.targetIdentityHash().equals(context.targetIdentityHash())) { + return StableCompletion.outcome( + MigrationStartupTargetVerification.DETERMINISTIC_MISMATCH); + } + VerificationHolder holder = new VerificationHolder(); + lease.withConnection(connection -> holder.set(inspector.inspect( + connection, bundle.application().metadataDatabase().kind(), fingerprint, deadline))); + return StableCompletion.outcome(holder.get()); + } catch (MigrationStartupReconciliationException failure) { + return StableCompletion.recovery(); + } catch (RuntimeException failure) { + return StableCompletion.recovery(); + } catch (Error fatal) { + return StableCompletion.fatal(fatal); + } + } + + private void settleAcquire( + VerificationContext context, + JdbcMetadataMigrationDeadline deadline) { + if (!state.hasPendingAcquire()) { + return; + } + state.requirePendingAcquire(context); + TargetJdbcFailedAcquireSettlement settlement; + try { + settlement = factory.settleFailedAcquire(deadline); + } catch (MetadataMigrationException timeoutFailure) { + throw timeoutFailure; + } catch (RuntimeException failure) { + throw recovery(); + } + if (settlement != TargetJdbcFailedAcquireSettlement.REUSABLE) { + throw recovery(); + } + state.clearPendingAcquire(); + } + + private InstallationFingerprint readFingerprint(JdbcMetadataMigrationDeadline deadline) { + check(deadline); + try { + Optional fingerprint = fingerprints.read(); + check(deadline); + return fingerprint.orElseThrow(CandidateBackedMigrationStartupTargetVerifier::recovery); + } catch (IOException failure) { + throw recovery(); + } + } + + private MigrationStartupTargetVerification classifyAcquire( + VerificationContext context, + TargetJdbcConnectionException failure) { + return switch (failure.code()) { + case TIMEOUT, UNAVAILABLE -> { + state.retainAcquire(context); + yield MigrationStartupTargetVerification.TRANSIENT_UNAVAILABLE; + } + case TARGET_MISMATCH -> MigrationStartupTargetVerification.DETERMINISTIC_MISMATCH; + case CLEANUP_REQUIRED -> { + state.retainAcquire(context); + throw recovery(); + } + case OPERATION_CONFLICT, FACTORY_CLOSED -> throw recovery(); + }; + } + + private static void check(JdbcMetadataMigrationDeadline deadline) { + if (Thread.currentThread().isInterrupted()) { + throw new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT); + } + deadline.remainingDuration(); + } + + private static MigrationStartupReconciliationException recovery() { + return new MigrationStartupReconciliationException(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupCurrentSchemaVerifier.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupCurrentSchemaVerifier.java new file mode 100644 index 0000000000..30ece3663a --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupCurrentSchemaVerifier.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.sql.SQLException; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Read-only boundary for proving the exact current target schema. */ +@FunctionalInterface +interface MigrationStartupCurrentSchemaVerifier { + + boolean isCurrent( + Connection connection, + MetadataDatabaseKind kind, + JdbcMetadataMigrationDeadline deadline) throws SQLException; +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupTargetInspector.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupTargetInspector.java new file mode 100644 index 0000000000..24f7bcb91e --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupTargetInspector.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.installation.InstallationFingerprint; + +/** Performs only current-schema and installation-identity reads on a caller-owned connection. */ +final class MigrationStartupTargetInspector { + + private static final String INSTALLATION_QUERY = + "SELECT id, installation_fingerprint, complete FROM hzb_installation"; + + private final MigrationStartupCurrentSchemaVerifier schema; + + MigrationStartupTargetInspector() { + this(new B206MigrationStartupCurrentSchemaVerifier()); + } + + MigrationStartupTargetInspector(MigrationStartupCurrentSchemaVerifier schema) { + this.schema = Objects.requireNonNull(schema, "schema"); + } + + MigrationStartupTargetVerification inspect( + Connection connection, + MetadataDatabaseKind kind, + InstallationFingerprint fingerprint, + JdbcMetadataMigrationDeadline deadline) { + Objects.requireNonNull(connection, "connection"); + Objects.requireNonNull(kind, "kind"); + Objects.requireNonNull(fingerprint, "fingerprint"); + Objects.requireNonNull(deadline, "deadline"); + TargetSchemaJdbcBudget budget = new TargetSchemaJdbcBudget(deadline); + try { + if (!schema.isCurrent(connection, kind, deadline)) { + return MigrationStartupTargetVerification.DETERMINISTIC_MISMATCH; + } + return installationMatches(connection, fingerprint, budget) + ? MigrationStartupTargetVerification.CONFIRMED + : MigrationStartupTargetVerification.DETERMINISTIC_MISMATCH; + } catch (MetadataMigrationException timeout) { + return MigrationStartupTargetVerification.TRANSIENT_UNAVAILABLE; + } catch (SQLException failure) { + if ("55000".equals(failure.getSQLState())) { + return MigrationStartupTargetVerification.DETERMINISTIC_MISMATCH; + } + if (TargetSchemaSqlFailure.invalidatesConnection(failure)) { + return MigrationStartupTargetVerification.TRANSIENT_UNAVAILABLE; + } + throw new MigrationStartupReconciliationException(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + } + + private boolean installationMatches( + Connection connection, + InstallationFingerprint fingerprint, + TargetSchemaJdbcBudget budget) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(INSTALLATION_QUERY)) { + budget.apply(statement); + try (ResultSet rows = statement.executeQuery()) { + budget.check(); + if (!rows.next()) { + return false; + } + budget.check(); + short id = rows.getShort("id"); + budget.check(); + String actualFingerprint = rows.getString("installation_fingerprint"); + budget.check(); + boolean complete = rows.getBoolean("complete"); + budget.check(); + boolean exact = id == 1 && complete && fingerprint.value().equals(actualFingerprint); + boolean anotherRow = rows.next(); + budget.check(); + return exact && !anotherRow; + } + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupTargetVerificationState.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupTargetVerificationState.java new file mode 100644 index 0000000000..6b6092563e --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupTargetVerificationState.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.CandidateRef; + +/** Owns exact retry state for one candidate-backed startup target verification. */ +final class MigrationStartupTargetVerificationState { + + private PendingLease pendingLease; + private VerificationContext pendingAcquire; + + VerificationContext context(CandidateRef candidate, String targetIdentityHash) { + return new VerificationContext(candidate, targetIdentityHash); + } + + boolean hasPendingLease() { + return pendingLease != null; + } + + MigrationStartupTargetVerification closePending(VerificationContext context) { + requireSame(pendingLease.context(), context); + try { + pendingLease.lease().close(); + } catch (RuntimeException failure) { + throw retainedFatalOrRecovery(); + } catch (Error fatal) { + pendingLease = pendingLease.withFatalUnlessPresent(fatal); + throw pendingLease.completion().fatal(); + } + StableCompletion completion = pendingLease.completion(); + pendingLease = null; + return completion.replay(); + } + + StableCompletion closePendingForShutdown() { + try { + pendingLease.lease().close(); + } catch (RuntimeException failure) { + throw retainedFatalOrRecovery(); + } catch (Error fatal) { + pendingLease = pendingLease.withFatalUnlessPresent(fatal); + throw pendingLease.completion().fatal(); + } + StableCompletion completion = pendingLease.completion(); + pendingLease = null; + return completion; + } + + void retainLease( + VerificationContext context, + TargetJdbcConnectionLease lease, + StableCompletion completion) { + pendingLease = new PendingLease(context, lease, completion); + } + + boolean hasPendingAcquire() { + return pendingAcquire != null; + } + + void requirePendingAcquire(VerificationContext context) { + requireSame(pendingAcquire, context); + } + + void retainAcquire(VerificationContext context) { + pendingAcquire = context; + } + + void clearPendingAcquire() { + pendingAcquire = null; + } + + private RuntimeException retainedFatalOrRecovery() { + Error fatal = pendingLease.completion().fatal(); + if (fatal != null) { + throw fatal; + } + return recovery(); + } + + private static void requireSame(VerificationContext expected, VerificationContext actual) { + if (!expected.equals(actual)) { + throw new MigrationStartupReconciliationException(SetupErrorCode.OPERATION_CONFLICT); + } + } + + record VerificationContext(CandidateRef candidate, String targetIdentityHash) { + + VerificationContext { + Objects.requireNonNull(candidate, "candidate"); + if (targetIdentityHash == null || !targetIdentityHash.matches("[0-9a-f]{64}")) { + throw new IllegalArgumentException("Invalid target identity"); + } + } + } + + private record PendingLease( + VerificationContext context, + TargetJdbcConnectionLease lease, + StableCompletion completion) { + + PendingLease withFatalUnlessPresent(Error fatal) { + return new PendingLease(context, lease, completion.withFatalUnlessPresent(fatal)); + } + } + + record StableCompletion( + MigrationStartupTargetVerification outcome, + SetupErrorCode recoveryCode, + Error fatal) { + + static StableCompletion outcome(MigrationStartupTargetVerification outcome) { + return new StableCompletion(Objects.requireNonNull(outcome, "outcome"), null, null); + } + + static StableCompletion recovery() { + return new StableCompletion(null, SetupErrorCode.CONFIG_RECOVERY_REQUIRED, null); + } + + static StableCompletion fatal(Error fatal) { + return new StableCompletion(null, null, Objects.requireNonNull(fatal, "fatal")); + } + + StableCompletion withFatalUnlessPresent(Error laterFatal) { + return fatal == null ? fatal(laterFatal) : this; + } + + MigrationStartupTargetVerification replay() { + if (fatal != null) { + throw fatal; + } + if (recoveryCode != null) { + throw new MigrationStartupReconciliationException(recoveryCode); + } + return outcome; + } + } + + static final class VerificationHolder { + + private MigrationStartupTargetVerification outcome; + + void set(MigrationStartupTargetVerification value) { + if (outcome != null) { + throw recovery(); + } + outcome = Objects.requireNonNull(value, "target verification"); + } + + MigrationStartupTargetVerification get() { + return Objects.requireNonNull(outcome, "target verification"); + } + } + + private static MigrationStartupReconciliationException recovery() { + return new MigrationStartupReconciliationException(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/CandidateBackedMigrationStartupTargetVerifierTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/CandidateBackedMigrationStartupTargetVerifierTest.java new file mode 100644 index 0000000000..af4518dddd --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/CandidateBackedMigrationStartupTargetVerifierTest.java @@ -0,0 +1,476 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.time.Duration; +import java.util.Optional; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.config.GreptimeEndpoints; +import org.apache.hertzbeat.manager.setup.config.GreptimeSettings; +import org.apache.hertzbeat.manager.setup.config.ManagedApplicationConfig; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigurationBundle; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.CandidateReader; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.CandidateRef; +import org.apache.hertzbeat.manager.setup.config.ManagedSecrets; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.apache.hertzbeat.manager.setup.installation.InstallationFingerprint; +import org.apache.hertzbeat.manager.setup.installation.LocalInstallationFingerprintStore; +import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class CandidateBackedMigrationStartupTargetVerifierTest { + + private static final CandidateRef CANDIDATE = new CandidateRef("operation-a", "candidate-generation"); + private static final String IDENTITY = "a".repeat(64); + private static final InstallationFingerprint FINGERPRINT = + new InstallationFingerprint("b".repeat(64)); + private static final MetadataDatabaseSettings SETTINGS = new MetadataDatabaseSettings( + MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://db.example/hertzbeat", "hertzbeat"); + + private ManagedMigrationConfigurationTransaction configuration; + private LocalInstallationFingerprintStore fingerprints; + private TargetJdbcConnectionFactory factory; + private TargetJdbcConnectionLease lease; + private MigrationStartupTargetInspector inspector; + private Connection connection; + private SecretValue borrowedPassword; + private ManagedConfigurationBundle bundle; + + @BeforeEach + void setUp() throws Exception { + configuration = mock(ManagedMigrationConfigurationTransaction.class); + fingerprints = mock(LocalInstallationFingerprintStore.class); + factory = mock(TargetJdbcConnectionFactory.class); + lease = mock(TargetJdbcConnectionLease.class); + inspector = mock(MigrationStartupTargetInspector.class); + connection = mock(Connection.class); + borrowedPassword = SecretValue.of("borrowed-password"); + bundle = new ManagedConfigurationBundle( + new ManagedApplicationConfig( + SETTINGS, + GreptimeSettings.anonymous( + new GreptimeEndpoints("greptime:4001", "http://greptime:4000"), "public")), + ManagedSecrets.withoutTelemetryPassword(borrowedPassword)); + when(fingerprints.read()).thenReturn(Optional.of(FINGERPRINT)); + when(configuration.readExact(eq(CANDIDATE), eq(IDENTITY), any())) + .thenAnswer(invocation -> invocation.>getArgument(2).read(bundle)); + when(factory.acquire(eq(SETTINGS), eq(borrowedPassword), any())).thenReturn(lease); + when(lease.targetIdentityHash()).thenReturn(IDENTITY); + doAnswer(invocation -> { + invocation.getArgument(0).execute(connection); + return null; + }).when(lease).withConnection(any()); + when(inspector.inspect(eq(connection), eq(MetadataDatabaseKind.POSTGRESQL), + eq(FINGERPRINT), any())).thenReturn(MigrationStartupTargetVerification.CONFIRMED); + } + + @Test + void borrowsExactCandidateSecretAndUsesOneRootDeadline() throws Exception { + CandidateBackedMigrationStartupTargetVerifier verifier = verifier(); + + assertThat(verifier.verify(CANDIDATE, IDENTITY)) + .isEqualTo(MigrationStartupTargetVerification.CONFIRMED); + + ArgumentCaptor acquisitionDeadline = + ArgumentCaptor.forClass(JdbcMetadataMigrationDeadline.class); + ArgumentCaptor inspectionDeadline = + ArgumentCaptor.forClass(JdbcMetadataMigrationDeadline.class); + verify(factory).acquire(eq(SETTINGS), eq(borrowedPassword), acquisitionDeadline.capture()); + verify(inspector).inspect(eq(connection), eq(MetadataDatabaseKind.POSTGRESQL), + eq(FINGERPRINT), inspectionDeadline.capture()); + assertThat(inspectionDeadline.getValue()).isSameAs(acquisitionDeadline.getValue()); + assertThat(borrowedPassword.copy()).containsExactly("borrowed-password".toCharArray()); + verify(lease).close(); + } + + @Test + void verifiedLeaseIdentityMismatchIsDeterministicAndSkipsSchemaRead() throws Exception { + when(lease.targetIdentityHash()).thenReturn("c".repeat(64)); + + assertThat(verifier().verify(CANDIDATE, IDENTITY)) + .isEqualTo(MigrationStartupTargetVerification.DETERMINISTIC_MISMATCH); + + verify(inspector, never()).inspect(any(), any(), any(), any()); + verify(lease).close(); + } + + @Test + void targetAcquireTimeoutIsTransientAndDoesNotConsumeBorrowedSecret() throws Exception { + when(factory.acquire(eq(SETTINGS), eq(borrowedPassword), any())) + .thenThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.TIMEOUT)); + + assertThat(verifier().verify(CANDIDATE, IDENTITY)) + .isEqualTo(MigrationStartupTargetVerification.TRANSIENT_UNAVAILABLE); + + assertThat(borrowedPassword.copy()).containsExactly("borrowed-password".toCharArray()); + verify(inspector, never()).inspect(any(), any(), any(), any()); + } + + @Test + void timeoutRetrySettlesTheExactFactoryAttemptBeforeCandidateIsReadAgain() throws Exception { + when(factory.acquire(eq(SETTINGS), eq(borrowedPassword), any())) + .thenThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.TIMEOUT)) + .thenReturn(lease); + when(factory.settleFailedAcquire(any())).thenReturn(TargetJdbcFailedAcquireSettlement.REUSABLE); + CandidateBackedMigrationStartupTargetVerifier verifier = verifier(); + + assertThat(verifier.verify(CANDIDATE, IDENTITY)) + .isEqualTo(MigrationStartupTargetVerification.TRANSIENT_UNAVAILABLE); + assertThat(verifier.verify(CANDIDATE, IDENTITY)) + .isEqualTo(MigrationStartupTargetVerification.CONFIRMED); + + InOrder retryOrder = inOrder(factory, configuration); + retryOrder.verify(configuration).readExact(eq(CANDIDATE), eq(IDENTITY), any()); + retryOrder.verify(factory).settleFailedAcquire(any()); + retryOrder.verify(configuration).readExact(eq(CANDIDATE), eq(IDENTITY), any()); + verify(factory, times(2)).acquire(eq(SETTINGS), eq(borrowedPassword), any()); + } + + @Test + void timeoutRetainsOperationOwnershipAndCloseDelegatesLateCleanupToFactory() throws Exception { + when(factory.acquire(eq(SETTINGS), eq(borrowedPassword), any())) + .thenThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.TIMEOUT)); + CandidateBackedMigrationStartupTargetVerifier verifier = verifier(); + assertThat(verifier.verify(CANDIDATE, IDENTITY)) + .isEqualTo(MigrationStartupTargetVerification.TRANSIENT_UNAVAILABLE); + + assertThatThrownBy(() -> verifier.verify( + new CandidateRef("operation-b", "candidate-generation"), "c".repeat(64))) + .isInstanceOfSatisfying(MigrationStartupReconciliationException.class, failure -> + assertThat(failure.errorCode()).isEqualTo(SetupErrorCode.OPERATION_CONFLICT)); + verifier.close(); + + verify(factory).close(); + verify(factory, never()).settleFailedAcquire(any()); + } + + @Test + void leaseCloseFailureRetainsExactOutcomeAndRetryDoesNotReacquire() throws Exception { + doThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED)) + .doNothing().when(lease).close(); + CandidateBackedMigrationStartupTargetVerifier verifier = verifier(); + + assertThatThrownBy(() -> verifier.verify(CANDIDATE, IDENTITY)) + .isInstanceOfSatisfying(MigrationStartupReconciliationException.class, failure -> + assertThat(failure.errorCode()) + .isEqualTo(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)); + assertThat(verifier.verify(CANDIDATE, IDENTITY)) + .isEqualTo(MigrationStartupTargetVerification.CONFIRMED); + + verify(configuration, times(1)).readExact(eq(CANDIDATE), eq(IDENTITY), any()); + verify(factory, times(1)).acquire(eq(SETTINGS), eq(borrowedPassword), any()); + verify(lease, times(2)).close(); + } + + @Test + void pendingLeasePrivateCloseFailureIsCauseFreeAndRetainsExactOutcome() throws Exception { + doThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED)) + .doThrow(new IllegalStateException("private close detail")) + .doNothing().when(lease).close(); + CandidateBackedMigrationStartupTargetVerifier verifier = verifier(); + + assertThatThrownBy(() -> verifier.verify(CANDIDATE, IDENTITY)) + .isInstanceOf(MigrationStartupReconciliationException.class); + assertThatThrownBy(() -> verifier.verify(CANDIDATE, IDENTITY)) + .isInstanceOfSatisfying(MigrationStartupReconciliationException.class, failure -> + assertThat(failure.errorCode()) + .isEqualTo(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)) + .hasNoCause() + .hasMessageNotContaining("private") + .hasMessageNotContaining("close detail"); + assertThat(verifier.verify(CANDIDATE, IDENTITY)) + .isEqualTo(MigrationStartupTargetVerification.CONFIRMED); + + verify(configuration, times(1)).readExact(eq(CANDIDATE), eq(IDENTITY), any()); + verify(factory, times(1)).acquire(eq(SETTINGS), eq(borrowedPassword), any()); + verify(lease, times(3)).close(); + } + + @Test + void settlementDeadlineRemainsTransientAndRetainsExactAcquireOwnership() throws Exception { + when(factory.acquire(eq(SETTINGS), eq(borrowedPassword), any())) + .thenThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.TIMEOUT)); + when(factory.settleFailedAcquire(any())) + .thenThrow(new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT)) + .thenReturn(TargetJdbcFailedAcquireSettlement.REUSABLE); + CandidateBackedMigrationStartupTargetVerifier verifier = verifier(); + + assertThat(verifier.verify(CANDIDATE, IDENTITY)) + .isEqualTo(MigrationStartupTargetVerification.TRANSIENT_UNAVAILABLE); + assertThat(verifier.verify(CANDIDATE, IDENTITY)) + .isEqualTo(MigrationStartupTargetVerification.TRANSIENT_UNAVAILABLE); + assertThat(verifier.verify(CANDIDATE, IDENTITY)) + .isEqualTo(MigrationStartupTargetVerification.TRANSIENT_UNAVAILABLE); + + verify(factory, times(2)).settleFailedAcquire(any()); + verify(configuration, times(2)).readExact(eq(CANDIDATE), eq(IDENTITY), any()); + } + + @Test + void privateFingerprintFailureIsCauseFreeRecoveryBeforeCandidateRead() throws Exception { + when(fingerprints.read()).thenThrow(new IllegalStateException("private fingerprint path")); + + assertThatThrownBy(() -> verifier().verify(CANDIDATE, IDENTITY)) + .isInstanceOfSatisfying(MigrationStartupReconciliationException.class, failure -> + assertThat(failure.errorCode()) + .isEqualTo(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)) + .hasNoCause() + .hasMessageNotContaining("private") + .hasMessageNotContaining("fingerprint path"); + + verify(configuration, never()).readExact(any(), any(), any()); + verify(factory, never()).acquire(any(), any(), any()); + } + + @Test + void candidateReadFailuresAreCauseFreeAndNeverAcquireTarget() throws Exception { + when(configuration.readExact(eq(CANDIDATE), eq(IDENTITY), any())) + .thenThrow(new IOException("private candidate path")) + .thenThrow(new IllegalStateException("private candidate state")); + CandidateBackedMigrationStartupTargetVerifier verifier = verifier(); + + assertRecoveryWithoutPrivateDetails( + () -> verifier.verify(CANDIDATE, IDENTITY), "candidate path"); + assertRecoveryWithoutPrivateDetails( + () -> verifier.verify(CANDIDATE, IDENTITY), "candidate state"); + + verify(factory, never()).acquire(any(), any(), any()); + } + + @Test + void interruptBeforeVerificationIsTransientAndPreservesFlag() throws Exception { + Thread.currentThread().interrupt(); + try { + assertThat(verifier().verify(CANDIDATE, IDENTITY)) + .isEqualTo(MigrationStartupTargetVerification.TRANSIENT_UNAVAILABLE); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + verify(fingerprints, never()).read(); + verify(configuration, never()).readExact(any(), any(), any()); + verify(factory, never()).acquire(any(), any(), any()); + } finally { + Thread.interrupted(); + } + } + + @Test + void inspectorPrivateRuntimeIsCauseFreeAndStillClosesLease() throws Exception { + when(inspector.inspect(eq(connection), eq(MetadataDatabaseKind.POSTGRESQL), + eq(FINGERPRINT), any())).thenThrow(new IllegalStateException("private SQL text")); + + assertRecoveryWithoutPrivateDetails( + () -> verifier().verify(CANDIDATE, IDENTITY), "private SQL text"); + + verify(lease).close(); + } + + @Test + void firstInspectorFatalRemainsPrimaryAcrossLeaseCloseFailureAndRetry() throws Exception { + AssertionError inspectorFatal = new AssertionError("inspector fatal"); + when(inspector.inspect(eq(connection), eq(MetadataDatabaseKind.POSTGRESQL), + eq(FINGERPRINT), any())).thenThrow(inspectorFatal); + doThrow(new AssertionError("later close fatal")).doNothing().when(lease).close(); + CandidateBackedMigrationStartupTargetVerifier verifier = verifier(); + + assertThatThrownBy(() -> verifier.verify(CANDIDATE, IDENTITY)).isSameAs(inspectorFatal); + assertThatThrownBy(() -> verifier.verify(CANDIDATE, IDENTITY)).isSameAs(inspectorFatal); + + verify(configuration, times(1)).readExact(eq(CANDIDATE), eq(IDENTITY), any()); + verify(factory, times(1)).acquire(eq(SETTINGS), eq(borrowedPassword), any()); + verify(lease, times(2)).close(); + } + + @Test + void inspectorFatalRemainsPrimaryWhenLeaseCloseNeedsRetry() throws Exception { + AssertionError inspectorFatal = new AssertionError("inspector fatal"); + when(inspector.inspect(eq(connection), eq(MetadataDatabaseKind.POSTGRESQL), + eq(FINGERPRINT), any())).thenThrow(inspectorFatal); + doThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED)) + .doThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED)) + .doNothing().when(lease).close(); + CandidateBackedMigrationStartupTargetVerifier verifier = verifier(); + + assertThatThrownBy(() -> verifier.verify(CANDIDATE, IDENTITY)).isSameAs(inspectorFatal); + assertThatThrownBy(() -> verifier.verify(CANDIDATE, IDENTITY)).isSameAs(inspectorFatal); + assertThatThrownBy(() -> verifier.verify(CANDIDATE, IDENTITY)).isSameAs(inspectorFatal); + + verify(configuration, times(1)).readExact(eq(CANDIDATE), eq(IDENTITY), any()); + verify(factory, times(1)).acquire(eq(SETTINGS), eq(borrowedPassword), any()); + verify(inspector, times(1)).inspect(any(), any(), any(), any()); + verify(lease, times(3)).close(); + } + + @Test + void shutdownKeepsInspectorFatalPrimaryUntilExactLeaseCleanupSucceeds() throws Exception { + AssertionError inspectorFatal = new AssertionError("inspector fatal"); + when(inspector.inspect(eq(connection), eq(MetadataDatabaseKind.POSTGRESQL), + eq(FINGERPRINT), any())).thenThrow(inspectorFatal); + doThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED)) + .doThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED)) + .doNothing().when(lease).close(); + CandidateBackedMigrationStartupTargetVerifier verifier = verifier(); + + assertThatThrownBy(() -> verifier.verify(CANDIDATE, IDENTITY)).isSameAs(inspectorFatal); + assertThatThrownBy(verifier::close).isSameAs(inspectorFatal); + assertThatThrownBy(verifier::close).isSameAs(inspectorFatal); + + verify(configuration, times(1)).readExact(eq(CANDIDATE), eq(IDENTITY), any()); + verify(factory, times(1)).acquire(eq(SETTINGS), eq(borrowedPassword), any()); + verify(inspector, times(1)).inspect(any(), any(), any(), any()); + verify(lease, times(3)).close(); + verify(factory).close(); + } + + @Test + void provisionalCleanupFailureRetainsExactOperationUntilFactorySettlement() throws Exception { + Connection provisional = mock(Connection.class); + DatabaseMetaData metadata = mock(DatabaseMetaData.class); + when(provisional.getAutoCommit()).thenReturn(true); + when(provisional.isReadOnly()).thenReturn(false); + when(provisional.getMetaData()).thenReturn(metadata); + when(metadata.getDatabaseProductName()).thenReturn("MySQL"); + when(metadata.getURL()).thenReturn("jdbc:mysql://other.example/hertzbeat"); + when(provisional.getCatalog()).thenReturn("hertzbeat"); + doThrow(new SQLException("private provisional cleanup")).doNothing() + .when(provisional).close(); + MetadataDatabaseSettings mysql = new MetadataDatabaseSettings( + MetadataDatabaseKind.MYSQL, + "jdbc:mysql://db.example/hertzbeat", "operator"); + ManagedConfigurationBundle mysqlBundle = new ManagedConfigurationBundle( + new ManagedApplicationConfig( + mysql, + GreptimeSettings.anonymous( + new GreptimeEndpoints("greptime:4001", "http://greptime:4000"), "public")), + ManagedSecrets.withoutTelemetryPassword(borrowedPassword)); + when(configuration.readExact(any(CandidateRef.class), anyString(), any())) + .thenAnswer(invocation -> invocation.>getArgument(2).read(mysqlBundle)); + TargetJdbcConnector connector = (target, username, password, deadline) -> provisional; + TargetJdbcConnectionFactory realFactory = new TargetJdbcConnectionFactory(connector, Runnable::run); + CandidateBackedMigrationStartupTargetVerifier verifier = new CandidateBackedMigrationStartupTargetVerifier( + configuration, fingerprints, realFactory, inspector, + Duration.ofSeconds(5), System::nanoTime); + + try { + assertThatThrownBy(() -> verifier.verify(CANDIDATE, IDENTITY)) + .isInstanceOf(MigrationStartupReconciliationException.class); + assertThatThrownBy(() -> verifier.verify( + new CandidateRef("operation-b", "candidate-generation"), "c".repeat(64))) + .isInstanceOfSatisfying(MigrationStartupReconciliationException.class, failure -> + assertThat(failure.errorCode()).isEqualTo(SetupErrorCode.OPERATION_CONFLICT)); + assertThatThrownBy(() -> verifier.verify(CANDIDATE, IDENTITY)) + .isInstanceOf(MigrationStartupReconciliationException.class); + + verify(configuration, times(1)).readExact(any(CandidateRef.class), anyString(), any()); + verify(provisional, times(2)).close(); + } finally { + verifier.close(); + } + } + + @Test + void settlementPrivateRuntimeIsCauseFreeAndKeepsExactAcquirePending() throws Exception { + when(factory.acquire(eq(SETTINGS), eq(borrowedPassword), any())) + .thenThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.TIMEOUT)) + .thenReturn(lease); + when(factory.settleFailedAcquire(any())) + .thenThrow(new IllegalStateException("private cleanup state")) + .thenReturn(TargetJdbcFailedAcquireSettlement.REUSABLE); + CandidateBackedMigrationStartupTargetVerifier verifier = verifier(); + + assertThat(verifier.verify(CANDIDATE, IDENTITY)) + .isEqualTo(MigrationStartupTargetVerification.TRANSIENT_UNAVAILABLE); + assertRecoveryWithoutPrivateDetails( + () -> verifier.verify(CANDIDATE, IDENTITY), "private cleanup state"); + assertThat(verifier.verify(CANDIDATE, IDENTITY)) + .isEqualTo(MigrationStartupTargetVerification.CONFIRMED); + + verify(factory, times(2)).settleFailedAcquire(any()); + verify(factory, times(2)).acquire(eq(SETTINGS), eq(borrowedPassword), any()); + } + + @Test + void shutdownCleanupPrivateRuntimeIsCauseFreeAndRemainsRetryable() throws Exception { + doThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED)) + .doThrow(new IllegalStateException("private shutdown close")) + .doNothing().when(lease).close(); + CandidateBackedMigrationStartupTargetVerifier verifier = verifier(); + assertThatThrownBy(() -> verifier.verify(CANDIDATE, IDENTITY)) + .isInstanceOf(MigrationStartupReconciliationException.class); + + assertRecoveryWithoutPrivateDetails(verifier::close, "private shutdown close"); + verifier.close(); + + verify(lease, times(3)).close(); + verify(factory).close(); + } + + @Test + void factoryShutdownPrivateRuntimeIsCauseFreeAndVerifierStaysClosed() { + doThrow(new IllegalStateException("private factory shutdown")).when(factory).close(); + CandidateBackedMigrationStartupTargetVerifier verifier = verifier(); + + assertRecoveryWithoutPrivateDetails(verifier::close, "private factory shutdown"); + assertRecoveryWithoutPrivateDetails( + () -> verifier.verify(CANDIDATE, IDENTITY), "private factory shutdown"); + + verify(factory).close(); + } + + @Test + void verifierAndRetryStateCannotRetainCredentialsOrTargetSettings() { + assertThat(CandidateBackedMigrationStartupTargetVerifier.class.getDeclaredFields()) + .extracting(Field::getType) + .doesNotContain(SecretValue.class) + .doesNotContain(MetadataDatabaseSettings.class); + assertThat(MigrationStartupTargetVerificationState.class.getDeclaredFields()) + .extracting(Field::getType) + .doesNotContain(SecretValue.class) + .doesNotContain(MetadataDatabaseSettings.class); + } + + private static void assertRecoveryWithoutPrivateDetails( + ThrowingCallable action, String privateDetail) { + assertThatThrownBy(action) + .isInstanceOfSatisfying(MigrationStartupReconciliationException.class, failure -> + assertThat(failure.errorCode()) + .isEqualTo(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)) + .hasNoCause() + .hasMessageNotContaining(privateDetail); + } + + private CandidateBackedMigrationStartupTargetVerifier verifier() { + return new CandidateBackedMigrationStartupTargetVerifier( + configuration, fingerprints, factory, inspector, + Duration.ofSeconds(5), () -> 0L); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupTargetInspectorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupTargetInspectorTest.java new file mode 100644 index 0000000000..4e01b2d9a3 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupTargetInspectorTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLTimeoutException; +import java.time.Duration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.installation.InstallationFingerprint; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class MigrationStartupTargetInspectorTest { + + private static final InstallationFingerprint FINGERPRINT = + new InstallationFingerprint("b".repeat(64)); + + private MigrationStartupCurrentSchemaVerifier schema; + private Connection connection; + private PreparedStatement statement; + private ResultSet rows; + private JdbcMetadataMigrationDeadline deadline; + + @BeforeEach + void setUp() throws Exception { + schema = mock(MigrationStartupCurrentSchemaVerifier.class); + connection = mock(Connection.class); + statement = mock(PreparedStatement.class); + rows = mock(ResultSet.class); + deadline = JdbcMetadataMigrationDeadline.start(Duration.ofSeconds(5), () -> 0L); + when(schema.isCurrent(connection, MetadataDatabaseKind.POSTGRESQL, deadline)).thenReturn(true); + when(connection.prepareStatement(any())).thenReturn(statement); + when(statement.executeQuery()).thenReturn(rows); + when(rows.next()).thenReturn(true, false); + when(rows.getShort("id")).thenReturn((short) 1); + when(rows.getString("installation_fingerprint")).thenReturn(FINGERPRINT.value()); + when(rows.getBoolean("complete")).thenReturn(true); + } + + @Test + void confirmsOnlyCurrentB206WithExactCompleteInstallationSingleton() throws Exception { + assertThat(inspector().inspect( + connection, MetadataDatabaseKind.POSTGRESQL, FINGERPRINT, deadline)) + .isEqualTo(MigrationStartupTargetVerification.CONFIRMED); + + verify(schema).isCurrent(connection, MetadataDatabaseKind.POSTGRESQL, deadline); + verify(statement).setQueryTimeout(5); + verify(rows).close(); + verify(statement).close(); + } + + @Test + void installationFingerprintMismatchIsDeterministic() throws Exception { + when(rows.getString("installation_fingerprint")).thenReturn("c".repeat(64)); + + assertThat(inspector().inspect( + connection, MetadataDatabaseKind.POSTGRESQL, FINGERPRINT, deadline)) + .isEqualTo(MigrationStartupTargetVerification.DETERMINISTIC_MISMATCH); + } + + @Test + void noncurrentSchemaIsDeterministicAndNeverReadsInstallation() throws Exception { + when(schema.isCurrent(connection, MetadataDatabaseKind.POSTGRESQL, deadline)).thenReturn(false); + + assertThat(inspector().inspect( + connection, MetadataDatabaseKind.POSTGRESQL, FINGERPRINT, deadline)) + .isEqualTo(MigrationStartupTargetVerification.DETERMINISTIC_MISMATCH); + + verify(connection, never()).prepareStatement(any()); + } + + @Test + void timeoutAndConnectionFailureRemainTransient() throws Exception { + when(schema.isCurrent(connection, MetadataDatabaseKind.POSTGRESQL, deadline)) + .thenThrow(new SQLTimeoutException("private timeout")) + .thenThrow(new SQLException("private connection", "08006")); + + assertThat(inspector().inspect( + connection, MetadataDatabaseKind.POSTGRESQL, FINGERPRINT, deadline)) + .isEqualTo(MigrationStartupTargetVerification.TRANSIENT_UNAVAILABLE); + assertThat(inspector().inspect( + connection, MetadataDatabaseKind.POSTGRESQL, FINGERPRINT, deadline)) + .isEqualTo(MigrationStartupTargetVerification.TRANSIENT_UNAVAILABLE); + } + + @Test + void permissionOrUnknownSqlFailureIsCauseFreeRecoveryNotTargetMismatch() throws Exception { + when(schema.isCurrent(connection, MetadataDatabaseKind.POSTGRESQL, deadline)) + .thenThrow(new SQLException("private permission", "42501")); + + assertThatThrownBy(() -> inspector().inspect( + connection, MetadataDatabaseKind.POSTGRESQL, FINGERPRINT, deadline)) + .isInstanceOfSatisfying(MigrationStartupReconciliationException.class, failure -> + assertThat(failure.errorCode()) + .isEqualTo(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)) + .hasNoCause() + .hasMessageNotContaining("private") + .hasMessageNotContaining("permission"); + } + + private MigrationStartupTargetInspector inspector() { + return new MigrationStartupTargetInspector(schema); + } +} From 649c6413f00b98a5adf9128b21dce8fc4be9c42a Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 16:52:45 +0800 Subject: [PATCH 60/71] Verify active migration startup configuration --- .../ManagedMigrationActiveConfiguration.java | 78 ++++++++++++ ...agedMigrationConfigurationTransaction.java | 13 ++ ...eBackedMigrationStartupTargetVerifier.java | 2 +- ...MigrationConfigurationTransactionTest.java | 113 ++++++++++++++++++ ...kedMigrationStartupTargetVerifierTest.java | 28 ++--- 5 files changed, 219 insertions(+), 15 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationActiveConfiguration.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationActiveConfiguration.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationActiveConfiguration.java new file mode 100644 index 0000000000..8ddc5c5ece --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationActiveConfiguration.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.config; + +import java.io.IOException; +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.CandidateReader; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.CandidateRef; + +/** Proves that the active aggregate is the complete configuration named by one candidate. */ +final class ManagedMigrationActiveConfiguration { + + private final ManagedApplicationConfigStore applications; + private final ManagedSecretStore secrets; + private final MigrationCandidateStore candidates; + + ManagedMigrationActiveConfiguration( + ManagedApplicationConfigStore applications, + ManagedSecretStore secrets, + MigrationCandidateStore candidates) { + this.applications = Objects.requireNonNull(applications, "applications"); + this.secrets = Objects.requireNonNull(secrets, "secrets"); + this.candidates = Objects.requireNonNull(candidates, "candidates"); + } + + T readExact( + CandidateRef reference, String targetIdentityHash, CandidateReader reader) throws IOException { + ActiveRead result = candidates.readExact(reference, targetIdentityHash, + candidate -> readActive(reference, candidate, reader)); + if (!result.exact()) { + throw new IOException("Active managed migration configuration requires recovery"); + } + return result.value(); + } + + private ActiveRead readActive( + CandidateRef reference, ManagedConfigurationBundle candidate, CandidateReader reader) { + CandidateRead activeApplication = applications.readActive(); + CandidateRead activeSecrets = secrets.readActive(); + try { + if (!ManagedConfigurationTransaction.validPair(activeApplication, activeSecrets) + || activeApplication.generation() + .filter(reference.candidateGeneration()::equals).isEmpty()) { + return ActiveRead.mismatch(); + } + ManagedConfigurationBundle active; + try { + active = new ManagedConfigurationBundle( + activeApplication.value().orElseThrow(), activeSecrets.value().orElseThrow()); + } catch (IllegalArgumentException failure) { + return ActiveRead.mismatch(); + } + if (!active.application().equals(candidate.application()) + || !active.secrets().equals(candidate.secrets())) { + return ActiveRead.mismatch(); + } + return ActiveRead.exact(reader.read(active)); + } finally { + ManagedConfigurationTransaction.close(activeSecrets); + } + } + + private record ActiveRead(boolean exact, T value) { + + static ActiveRead exact(T value) { + return new ActiveRead<>(true, value); + } + + static ActiveRead mismatch() { + return new ActiveRead<>(false, null); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransaction.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransaction.java index bc277bd3e2..9379e85266 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransaction.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransaction.java @@ -25,6 +25,7 @@ public final class ManagedMigrationConfigurationTransaction { private final MigrationCandidateStore store; private final ManagedMigrationActivation activation; private final ManagedMetadataTargetStage metadataTargetStage; + private final ManagedMigrationActiveConfiguration activeConfiguration; /** Creates the production migration candidate transaction. */ public ManagedMigrationConfigurationTransaction(Path installationRoot) { @@ -34,6 +35,7 @@ public final class ManagedMigrationConfigurationTransaction { FileManagedSecretStore secrets = new FileManagedSecretStore(installationRoot); activation = new ManagedMigrationActivation(applications, secrets); metadataTargetStage = new ManagedMetadataTargetStage(applications, secrets, store::stage); + activeConfiguration = new ManagedMigrationActiveConfiguration(applications, secrets, store); } /** Stages a metadata-only target over the exact active H2 managed configuration. */ @@ -95,6 +97,17 @@ public final class ManagedMigrationConfigurationTransaction { return lock.execute(() -> store.readExact(reference, expectedTargetIdentityHash, reader)); } + /** Reads only when active application and secrets exactly equal the identity-bound candidate. */ + public T readExactActive( + CandidateRef reference, String expectedTargetIdentityHash, CandidateReader reader) + throws IOException { + Objects.requireNonNull(reference, "reference"); + requireIdentityHash(expectedTargetIdentityHash); + Objects.requireNonNull(reader, "reader"); + return lock.execute(() -> activeConfiguration.readExact( + reference, expectedTargetIdentityHash, reader)); + } + /** Removes only the operation-and-generation scoped candidate named by the reference. */ public DiscardOutcome discardExact(CandidateRef reference) throws IOException { Objects.requireNonNull(reference, "reference"); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/CandidateBackedMigrationStartupTargetVerifier.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/CandidateBackedMigrationStartupTargetVerifier.java index e91b116da5..5a6f0be003 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/CandidateBackedMigrationStartupTargetVerifier.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/CandidateBackedMigrationStartupTargetVerifier.java @@ -66,7 +66,7 @@ final class CandidateBackedMigrationStartupTargetVerifier } settleAcquire(context, deadline); InstallationFingerprint fingerprint = readFingerprint(deadline); - return configuration.readExact(candidate, targetIdentityHash, + return configuration.readExactActive(candidate, targetIdentityHash, bundle -> verifyBundle(context, bundle, fingerprint, deadline)); } catch (IOException failure) { throw recovery(); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransactionTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransactionTest.java index 2f2344e7a8..c27aa4e4bf 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransactionTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/config/ManagedMigrationConfigurationTransactionTest.java @@ -21,6 +21,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; @@ -122,6 +123,86 @@ class ManagedMigrationConfigurationTransactionTest { candidate -> "unreachable")); } + @Test + void readsOnlyTheExactActiveCandidateAggregateAndClosesItsSecrets() throws Exception { + ManagedMigrationConfigurationTransaction migration = activatedCandidate(); + ManagedMigrationConfigurationTransaction.CandidateRef ref = reference(); + AtomicReference observed = new AtomicReference<>(); + + assertEquals("jdbc:postgresql://db/next", migration.readExactActive(ref, IDENTITY, active -> { + observed.set(active.secrets()); + assertEquals("database-next", new String(active.secrets().metadataDatabasePassword().copy())); + return active.application().metadataDatabase().jdbcUrl(); + })); + + assertTrue(new String(observed.get().metadataDatabasePassword().copy()) + .chars().allMatch(value -> value == 0)); + assertTrue(new String(observed.get().telemetryPassword().orElseThrow().copy()) + .chars().allMatch(value -> value == 0)); + assertTrue(new String(observed.get().mailPassword().orElseThrow().copy()) + .chars().allMatch(value -> value == 0)); + } + + @Test + void exactActiveReadRejectsLaterGenerationWithoutInvokingReader() throws Exception { + ManagedMigrationConfigurationTransaction migration = activatedCandidate(); + try (ManagedConfigurationBundle later = bundle("later")) { + assertEquals(ManagedConfigurationTransaction.Outcome.APPLIED, + new ManagedConfigurationTransaction(installationRoot).apply(later)); + } + AtomicBoolean invoked = new AtomicBoolean(); + + assertThrows(IOException.class, () -> migration.readExactActive(reference(), IDENTITY, active -> { + invoked.set(true); + return null; + })); + + assertTrue(!invoked.get()); + assertEquals(ManagedMigrationConfigurationTransaction.CandidateState.READY, + migration.inspect(reference()).state()); + } + + @Test + void exactActiveReadRejectsSameGenerationWithDifferentTargetAndPassword() throws Exception { + ManagedMigrationConfigurationTransaction migration = activatedCandidate(); + try (ManagedConfigurationBundle different = bundle("different")) { + writeActive(different, CANDIDATE); + } + AtomicBoolean invoked = new AtomicBoolean(); + + assertThrows(IOException.class, () -> migration.readExactActive(reference(), IDENTITY, active -> { + invoked.set(true); + return null; + })); + + assertTrue(!invoked.get()); + assertEquals(ManagedMigrationConfigurationTransaction.CandidateState.READY, + migration.inspect(reference()).state()); + } + + @Test + void exactActiveReadRejectsAggregateInvalidSecretsWithoutInvokingReader() throws Exception { + ManagedMigrationConfigurationTransaction migration = activatedCandidate(); + ManagedSecrets incomplete = ManagedSecrets.withoutTelemetryPassword(SecretValue.of("database-next")); + byte[] encoded = new SecretConfigDocumentCodec().encode(incomplete, CANDIDATE); + try { + Files.write(installationRoot.resolve("data/config/managed-secrets.properties"), encoded); + } finally { + Arrays.fill(encoded, (byte) 0); + incomplete.close(); + } + AtomicBoolean invoked = new AtomicBoolean(); + + assertThrows(IOException.class, () -> migration.readExactActive(reference(), IDENTITY, active -> { + invoked.set(true); + return null; + })); + + assertTrue(!invoked.get()); + assertEquals(ManagedMigrationConfigurationTransaction.CandidateState.READY, + migration.inspect(reference()).state()); + } + @Test void closesDecodedSecretsWhenTheSynchronousReaderFails() throws Exception { new ManagedConfigurationTransaction(installationRoot).apply(bundle("base")); @@ -467,6 +548,38 @@ class ManagedMigrationConfigurationTransactionTest { .resolve(OPERATION).resolve(generation); } + private ManagedMigrationConfigurationTransaction activatedCandidate() throws Exception { + try (ManagedConfigurationBundle base = bundle("base"); + ManagedConfigurationBundle next = bundle("next")) { + assertEquals(ManagedConfigurationTransaction.Outcome.APPLIED, + new ManagedConfigurationTransaction(installationRoot).apply(base)); + String actualBase = new FileManagedApplicationConfigStore(installationRoot) + .readActive().generation().orElseThrow(); + ManagedMigrationConfigurationTransaction migration = + new ManagedMigrationConfigurationTransaction(installationRoot); + migration.stage(OPERATION, CANDIDATE, actualBase, IDENTITY, next); + assertEquals(ManagedMigrationConfigurationTransaction.ActivationOutcome.ACTIVATED, + migration.activateExact(reference(), IDENTITY)); + return migration; + } + } + + private ManagedMigrationConfigurationTransaction.CandidateRef reference() { + return new ManagedMigrationConfigurationTransaction.CandidateRef(OPERATION, CANDIDATE); + } + + private void writeActive(ManagedConfigurationBundle active, String generation) throws IOException { + byte[] application = new ApplicationConfigDocumentCodec().encode(active.application(), generation); + byte[] secrets = new SecretConfigDocumentCodec().encode(active.secrets(), generation); + try { + Files.write(installationRoot.resolve("data/config/managed-application.yml"), application); + Files.write(installationRoot.resolve("data/config/managed-secrets.properties"), secrets); + } finally { + Arrays.fill(application, (byte) 0); + Arrays.fill(secrets, (byte) 0); + } + } + private static ManagedConfigurationBundle bundle(String suffix) { return new ManagedConfigurationBundle(configuration(suffix), new ManagedSecrets(SecretValue.of("database-" + suffix), diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/CandidateBackedMigrationStartupTargetVerifierTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/CandidateBackedMigrationStartupTargetVerifierTest.java index af4518dddd..2c759a8d5b 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/CandidateBackedMigrationStartupTargetVerifierTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/CandidateBackedMigrationStartupTargetVerifierTest.java @@ -83,7 +83,7 @@ class CandidateBackedMigrationStartupTargetVerifierTest { new GreptimeEndpoints("greptime:4001", "http://greptime:4000"), "public")), ManagedSecrets.withoutTelemetryPassword(borrowedPassword)); when(fingerprints.read()).thenReturn(Optional.of(FINGERPRINT)); - when(configuration.readExact(eq(CANDIDATE), eq(IDENTITY), any())) + when(configuration.readExactActive(eq(CANDIDATE), eq(IDENTITY), any())) .thenAnswer(invocation -> invocation.>getArgument(2).read(bundle)); when(factory.acquire(eq(SETTINGS), eq(borrowedPassword), any())).thenReturn(lease); when(lease.targetIdentityHash()).thenReturn(IDENTITY); @@ -151,9 +151,9 @@ class CandidateBackedMigrationStartupTargetVerifierTest { .isEqualTo(MigrationStartupTargetVerification.CONFIRMED); InOrder retryOrder = inOrder(factory, configuration); - retryOrder.verify(configuration).readExact(eq(CANDIDATE), eq(IDENTITY), any()); + retryOrder.verify(configuration).readExactActive(eq(CANDIDATE), eq(IDENTITY), any()); retryOrder.verify(factory).settleFailedAcquire(any()); - retryOrder.verify(configuration).readExact(eq(CANDIDATE), eq(IDENTITY), any()); + retryOrder.verify(configuration).readExactActive(eq(CANDIDATE), eq(IDENTITY), any()); verify(factory, times(2)).acquire(eq(SETTINGS), eq(borrowedPassword), any()); } @@ -188,7 +188,7 @@ class CandidateBackedMigrationStartupTargetVerifierTest { assertThat(verifier.verify(CANDIDATE, IDENTITY)) .isEqualTo(MigrationStartupTargetVerification.CONFIRMED); - verify(configuration, times(1)).readExact(eq(CANDIDATE), eq(IDENTITY), any()); + verify(configuration, times(1)).readExactActive(eq(CANDIDATE), eq(IDENTITY), any()); verify(factory, times(1)).acquire(eq(SETTINGS), eq(borrowedPassword), any()); verify(lease, times(2)).close(); } @@ -212,7 +212,7 @@ class CandidateBackedMigrationStartupTargetVerifierTest { assertThat(verifier.verify(CANDIDATE, IDENTITY)) .isEqualTo(MigrationStartupTargetVerification.CONFIRMED); - verify(configuration, times(1)).readExact(eq(CANDIDATE), eq(IDENTITY), any()); + verify(configuration, times(1)).readExactActive(eq(CANDIDATE), eq(IDENTITY), any()); verify(factory, times(1)).acquire(eq(SETTINGS), eq(borrowedPassword), any()); verify(lease, times(3)).close(); } @@ -234,7 +234,7 @@ class CandidateBackedMigrationStartupTargetVerifierTest { .isEqualTo(MigrationStartupTargetVerification.TRANSIENT_UNAVAILABLE); verify(factory, times(2)).settleFailedAcquire(any()); - verify(configuration, times(2)).readExact(eq(CANDIDATE), eq(IDENTITY), any()); + verify(configuration, times(2)).readExactActive(eq(CANDIDATE), eq(IDENTITY), any()); } @Test @@ -249,13 +249,13 @@ class CandidateBackedMigrationStartupTargetVerifierTest { .hasMessageNotContaining("private") .hasMessageNotContaining("fingerprint path"); - verify(configuration, never()).readExact(any(), any(), any()); + verify(configuration, never()).readExactActive(any(), any(), any()); verify(factory, never()).acquire(any(), any(), any()); } @Test void candidateReadFailuresAreCauseFreeAndNeverAcquireTarget() throws Exception { - when(configuration.readExact(eq(CANDIDATE), eq(IDENTITY), any())) + when(configuration.readExactActive(eq(CANDIDATE), eq(IDENTITY), any())) .thenThrow(new IOException("private candidate path")) .thenThrow(new IllegalStateException("private candidate state")); CandidateBackedMigrationStartupTargetVerifier verifier = verifier(); @@ -276,7 +276,7 @@ class CandidateBackedMigrationStartupTargetVerifierTest { .isEqualTo(MigrationStartupTargetVerification.TRANSIENT_UNAVAILABLE); assertThat(Thread.currentThread().isInterrupted()).isTrue(); verify(fingerprints, never()).read(); - verify(configuration, never()).readExact(any(), any(), any()); + verify(configuration, never()).readExactActive(any(), any(), any()); verify(factory, never()).acquire(any(), any(), any()); } finally { Thread.interrupted(); @@ -305,7 +305,7 @@ class CandidateBackedMigrationStartupTargetVerifierTest { assertThatThrownBy(() -> verifier.verify(CANDIDATE, IDENTITY)).isSameAs(inspectorFatal); assertThatThrownBy(() -> verifier.verify(CANDIDATE, IDENTITY)).isSameAs(inspectorFatal); - verify(configuration, times(1)).readExact(eq(CANDIDATE), eq(IDENTITY), any()); + verify(configuration, times(1)).readExactActive(eq(CANDIDATE), eq(IDENTITY), any()); verify(factory, times(1)).acquire(eq(SETTINGS), eq(borrowedPassword), any()); verify(lease, times(2)).close(); } @@ -324,7 +324,7 @@ class CandidateBackedMigrationStartupTargetVerifierTest { assertThatThrownBy(() -> verifier.verify(CANDIDATE, IDENTITY)).isSameAs(inspectorFatal); assertThatThrownBy(() -> verifier.verify(CANDIDATE, IDENTITY)).isSameAs(inspectorFatal); - verify(configuration, times(1)).readExact(eq(CANDIDATE), eq(IDENTITY), any()); + verify(configuration, times(1)).readExactActive(eq(CANDIDATE), eq(IDENTITY), any()); verify(factory, times(1)).acquire(eq(SETTINGS), eq(borrowedPassword), any()); verify(inspector, times(1)).inspect(any(), any(), any(), any()); verify(lease, times(3)).close(); @@ -344,7 +344,7 @@ class CandidateBackedMigrationStartupTargetVerifierTest { assertThatThrownBy(verifier::close).isSameAs(inspectorFatal); assertThatThrownBy(verifier::close).isSameAs(inspectorFatal); - verify(configuration, times(1)).readExact(eq(CANDIDATE), eq(IDENTITY), any()); + verify(configuration, times(1)).readExactActive(eq(CANDIDATE), eq(IDENTITY), any()); verify(factory, times(1)).acquire(eq(SETTINGS), eq(borrowedPassword), any()); verify(inspector, times(1)).inspect(any(), any(), any(), any()); verify(lease, times(3)).close(); @@ -372,7 +372,7 @@ class CandidateBackedMigrationStartupTargetVerifierTest { GreptimeSettings.anonymous( new GreptimeEndpoints("greptime:4001", "http://greptime:4000"), "public")), ManagedSecrets.withoutTelemetryPassword(borrowedPassword)); - when(configuration.readExact(any(CandidateRef.class), anyString(), any())) + when(configuration.readExactActive(any(CandidateRef.class), anyString(), any())) .thenAnswer(invocation -> invocation.>getArgument(2).read(mysqlBundle)); TargetJdbcConnector connector = (target, username, password, deadline) -> provisional; TargetJdbcConnectionFactory realFactory = new TargetJdbcConnectionFactory(connector, Runnable::run); @@ -390,7 +390,7 @@ class CandidateBackedMigrationStartupTargetVerifierTest { assertThatThrownBy(() -> verifier.verify(CANDIDATE, IDENTITY)) .isInstanceOf(MigrationStartupReconciliationException.class); - verify(configuration, times(1)).readExact(any(CandidateRef.class), anyString(), any()); + verify(configuration, times(1)).readExactActive(any(CandidateRef.class), anyString(), any()); verify(provisional, times(2)).close(); } finally { verifier.close(); From 64a153f8e8b948bec80b02f2315524816e449582 Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 17:12:51 +0800 Subject: [PATCH 61/71] Coordinate managed migration startup recovery --- ...anagedMigrationStartupRecoveryRuntime.java | 54 +++ .../workflow/FileMigrationOperationStore.java | 13 + .../ManagedMigrationStartupReconciler.java | 7 +- ...edMigrationStartupRecoveryDisposition.java | 15 + ...anagedMigrationStartupRecoveryRuntime.java | 17 + ...anagedMigrationStartupRecoverySession.java | 147 +++++++ .../workflow/MigrationStartupSnapshots.java | 8 +- ...ManagedMigrationStartupReconcilerTest.java | 31 ++ ...edMigrationStartupRecoverySessionTest.java | 370 ++++++++++++++++++ 9 files changed, 657 insertions(+), 5 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultManagedMigrationStartupRecoveryRuntime.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupRecoveryDisposition.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupRecoveryRuntime.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupRecoverySession.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupRecoverySessionTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultManagedMigrationStartupRecoveryRuntime.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultManagedMigrationStartupRecoveryRuntime.java new file mode 100644 index 0000000000..468d66639a --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultManagedMigrationStartupRecoveryRuntime.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.nio.file.Path; +import java.security.SecureRandom; +import java.time.Clock; +import java.time.Duration; +import java.util.Objects; +import java.util.concurrent.Executor; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.installation.LocalInstallationFingerprintStore; + +/** Owns the JDBC verifier and managed reconciler dependencies across gated startup retries. */ +final class DefaultManagedMigrationStartupRecoveryRuntime + implements ManagedMigrationStartupRecoveryRuntime { + + private final FileMigrationOperationStore store; + private final ManagedMigrationConfigurationTransaction configuration; + private final CandidateBackedMigrationStartupTargetVerifier verifier; + private final Clock clock; + + DefaultManagedMigrationStartupRecoveryRuntime( + Path root, + FileMigrationOperationStore store, + Duration timeout, + Executor abortExecutor) { + this.store = Objects.requireNonNull(store, "store"); + configuration = new ManagedMigrationConfigurationTransaction(root); + TargetJdbcConnectionFactory factory = new TargetJdbcConnectionFactory(abortExecutor); + LocalInstallationFingerprintStore fingerprints = new LocalInstallationFingerprintStore( + root, root.resolve("data/config/.installation-fingerprint"), new SecureRandom()); + verifier = new CandidateBackedMigrationStartupTargetVerifier( + configuration, fingerprints, factory, new MigrationStartupTargetInspector(), + timeout, System::nanoTime); + clock = Clock.systemUTC(); + } + + @Override + public MigrationStartupReconciliation reconcile(DurableCutoverDraft draft) { + return new ManagedMigrationStartupReconciler( + draft, store, configuration, verifier, clock).reconcile(); + } + + @Override + public void close() { + verifier.close(); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java index a2b63e6449..71a2178f6b 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java @@ -115,6 +115,19 @@ public final class FileMigrationOperationStore implements MigrationOperationStor }); } + /** Selects the only nonterminal startup record under the operation-store lock. */ + Optional selectUniqueNonterminalForStartup() { + return locked(() -> { + List active = read().stream() + .filter(snapshot -> !snapshot.terminal()) + .toList(); + if (active.size() > 1) { + throw failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + return active.stream().findFirst(); + }); + } + @Override public List history() { return locked(() -> List.copyOf(read())); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupReconciler.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupReconciler.java index 5a10119c5c..add713cea7 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupReconciler.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupReconciler.java @@ -73,7 +73,7 @@ final class ManagedMigrationStartupReconciler { return MigrationStartupReconciliation.ALREADY_ROLLED_BACK_RESTART_REQUIRED; } current = convergeActivation(current); - if (isRestartRollback(current)) { + if (isStartupRollback(current)) { return rollback(current); } if (!new RetainedManagedActivationSnapshots(current).awaitingRestart()) { @@ -135,10 +135,11 @@ final class ManagedMigrationStartupReconciler { return MigrationStartupReconciliation.ROLLED_BACK_RESTART_REQUIRED; } - private boolean isRestartRollback(MigrationOperationSnapshot current) { + private boolean isStartupRollback(MigrationOperationSnapshot current) { return current.state() == MigrationOperationState.RUNNING && current.stage() == MigrationStage.ROLLING_BACK - && current.rollbackOrigin() == MigrationRollbackOrigin.RESTART_FAILURE; + && (current.rollbackOrigin() == MigrationRollbackOrigin.ACTIVATION_FAILURE + || current.rollbackOrigin() == MigrationRollbackOrigin.RESTART_FAILURE); } private void requireExactDraft(MigrationOperationSnapshot current) { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupRecoveryDisposition.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupRecoveryDisposition.java new file mode 100644 index 0000000000..fbd4d0d63f --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupRecoveryDisposition.java @@ -0,0 +1,15 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +/** Secret-free manager-side startup recovery disposition. */ +public enum ManagedMigrationStartupRecoveryDisposition { + NO_MIGRATION, + GATED_RECOVERY, + RELOAD_FULL_GATED +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupRecoveryRuntime.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupRecoveryRuntime.java new file mode 100644 index 0000000000..8091179c79 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupRecoveryRuntime.java @@ -0,0 +1,17 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +/** Long-lived reconciliation runtime retained by one startup recovery session. */ +interface ManagedMigrationStartupRecoveryRuntime extends AutoCloseable { + + MigrationStartupReconciliation reconcile(DurableCutoverDraft draft); + + @Override + void close(); +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupRecoverySession.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupRecoverySession.java new file mode 100644 index 0000000000..e3c677e4cf --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupRecoverySession.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.IOException; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.Executor; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; + +/** + * Holds one exact migration startup recovery binding across retries without Spring or persistence beans. + * The supplied abort executor is caller-owned and must outlive this session's late JDBC cleanup. + */ +public final class ManagedMigrationStartupRecoverySession implements AutoCloseable { + + private final FileMigrationOperationStore store; + private final ManagedMigrationStartupRecoveryRuntime runtime; + private Selection selection; + private boolean closed; + + public ManagedMigrationStartupRecoverySession( + Path installationRoot, Duration verificationTimeout, Executor abortExecutor) { + Path canonicalRoot = canonicalRoot(installationRoot); + store = new FileMigrationOperationStore(canonicalRoot); + runtime = new DefaultManagedMigrationStartupRecoveryRuntime( + canonicalRoot, store, + Objects.requireNonNull(verificationTimeout, "verificationTimeout"), + Objects.requireNonNull(abortExecutor, "abortExecutor")); + } + + ManagedMigrationStartupRecoverySession( + Path installationRoot, + FileMigrationOperationStore store, + ManagedMigrationStartupRecoveryRuntime runtime) { + canonicalRoot(installationRoot); + this.store = Objects.requireNonNull(store, "store"); + this.runtime = Objects.requireNonNull(runtime, "runtime"); + } + + /** Reconciles only the operation selected and bound by this session's first observation. */ + public synchronized ManagedMigrationStartupRecoveryDisposition reconcile() { + if (closed) { + return ManagedMigrationStartupRecoveryDisposition.GATED_RECOVERY; + } + if (selection == null) { + selection = selectOnce(); + } + if (selection.disposition() != null) { + return selection.disposition(); + } + try { + ManagedMigrationStartupRecoveryDisposition disposition = + map(runtime.reconcile(selection.draft())); + if (disposition == ManagedMigrationStartupRecoveryDisposition.RELOAD_FULL_GATED) { + selection = Selection.fixed(disposition); + } + return disposition; + } catch (MigrationStartupReconciliationException | MigrationOperationStoreException failure) { + return ManagedMigrationStartupRecoveryDisposition.GATED_RECOVERY; + } catch (RuntimeException failure) { + return ManagedMigrationStartupRecoveryDisposition.GATED_RECOVERY; + } + } + + @Override + public synchronized void close() { + if (closed) { + return; + } + runtime.close(); + closed = true; + } + + private Selection selectOnce() { + try { + Optional selected = store.selectUniqueNonterminalForStartup(); + if (selected.isEmpty()) { + return Selection.fixed(ManagedMigrationStartupRecoveryDisposition.NO_MIGRATION); + } + MigrationOperationSnapshot snapshot = selected.orElseThrow(); + if (!actionable(snapshot)) { + return Selection.fixed(ManagedMigrationStartupRecoveryDisposition.GATED_RECOVERY); + } + return Selection.bound(new DurableCutoverDraft( + snapshot.operationId(), snapshot.target(), snapshot.applyMode(), + snapshot.createdAt(), snapshot.startedAt(), snapshot.managedCandidateGeneration())); + } catch (MigrationOperationStoreException | IllegalArgumentException failure) { + return Selection.fixed(ManagedMigrationStartupRecoveryDisposition.GATED_RECOVERY); + } + } + + private boolean actionable(MigrationOperationSnapshot snapshot) { + if (snapshot.applyMode() != ApplyMode.MANAGED_WRITE) { + return false; + } + boolean awaitingRestart = snapshot.state() == MigrationOperationState.AWAITING_RESTART + && snapshot.stage() == MigrationStage.AWAITING_RESTART + || snapshot.state() == MigrationOperationState.RUNNING + && snapshot.stage() == MigrationStage.ACTIVATING; + boolean startupRollback = snapshot.state() == MigrationOperationState.RUNNING + && snapshot.stage() == MigrationStage.ROLLING_BACK + && (snapshot.rollbackOrigin() == MigrationRollbackOrigin.ACTIVATION_FAILURE + || snapshot.rollbackOrigin() == MigrationRollbackOrigin.RESTART_FAILURE); + return awaitingRestart || startupRollback; + } + + private ManagedMigrationStartupRecoveryDisposition map(MigrationStartupReconciliation outcome) { + return switch (Objects.requireNonNull(outcome, "reconciliation outcome")) { + case GATED, NO_MIGRATION -> ManagedMigrationStartupRecoveryDisposition.GATED_RECOVERY; + case SUCCEEDED, ALREADY_SUCCEEDED, ROLLED_BACK_RESTART_REQUIRED, + ALREADY_ROLLED_BACK_RESTART_REQUIRED -> + ManagedMigrationStartupRecoveryDisposition.RELOAD_FULL_GATED; + }; + } + + private static Path canonicalRoot(Path root) { + try { + return SecureSetupFile.prepareTrustedRoot(Objects.requireNonNull(root, "installationRoot")); + } catch (IOException | RuntimeException failure) { + throw new IllegalArgumentException("Migration startup recovery root is unsafe"); + } + } + + private record Selection( + DurableCutoverDraft draft, + ManagedMigrationStartupRecoveryDisposition disposition) { + + static Selection bound(DurableCutoverDraft draft) { + return new Selection(Objects.requireNonNull(draft, "draft"), null); + } + + static Selection fixed(ManagedMigrationStartupRecoveryDisposition disposition) { + return new Selection(null, Objects.requireNonNull(disposition, "disposition")); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupSnapshots.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupSnapshots.java index d1a771e857..c2efe56609 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupSnapshots.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationStartupSnapshots.java @@ -34,9 +34,13 @@ final class MigrationStartupSnapshots { } MigrationOperationSnapshot rolledBack(Instant completedAt) { + MigrationRollbackOrigin rollbackOrigin = source.rollbackOrigin(); + if (rollbackOrigin != MigrationRollbackOrigin.ACTIVATION_FAILURE + && rollbackOrigin != MigrationRollbackOrigin.RESTART_FAILURE) { + throw new IllegalArgumentException("Unsupported startup rollback origin"); + } return snapshot(MigrationOperationState.ROLLED_BACK, MigrationStage.ROLLED_BACK, - completedAt, SetupErrorCode.RESTART_FAILED, - MigrationRollbackOrigin.RESTART_FAILURE, 0); + completedAt, rollbackOrigin.errorCode(), rollbackOrigin, 0); } private MigrationOperationSnapshot snapshot( diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupReconcilerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupReconcilerTest.java index 1f41d07714..f52484a76b 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupReconcilerTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupReconcilerTest.java @@ -204,6 +204,28 @@ class ManagedMigrationStartupReconcilerTest { verify(configuration, times(2)).rollbackExact(CANDIDATE, IDENTITY); } + @Test + void activationFailureRollbackConvergesAndReplaysItsExactTerminalCause() throws Exception { + MigrationOperationSnapshot rollingBack = activationRollback(current()); + store.compareAndTransition( + OPERATION, MigrationOperationState.READY_TO_ACTIVATE, rollingBack); + when(configuration.rollbackExact(CANDIDATE, IDENTITY)) + .thenReturn(RollbackOutcome.ROLLED_BACK); + ManagedMigrationStartupReconciler reconciler = reconciler(); + + assertThat(reconciler.reconcile()) + .isEqualTo(MigrationStartupReconciliation.ROLLED_BACK_RESTART_REQUIRED); + assertThat(reconciler.reconcile()) + .isEqualTo(MigrationStartupReconciliation.ALREADY_ROLLED_BACK_RESTART_REQUIRED); + + MigrationOperationSnapshot terminal = current(); + assertThat(terminal.state()).isEqualTo(MigrationOperationState.ROLLED_BACK); + assertThat(terminal.rollbackOrigin()).isEqualTo(MigrationRollbackOrigin.ACTIVATION_FAILURE); + assertThat(terminal.errorCode()).isEqualTo(SetupErrorCode.MIGRATION_ACTIVATION_FAILED); + verify(configuration, times(1)).rollbackExact(CANDIDATE, IDENTITY); + verifyNoInteractions(verifier); + } + @Test void targetVerifierErrorRemainsPrimaryAndDoesNotChangeJournal() { seedAwaitingRestart(); @@ -283,4 +305,13 @@ class ManagedMigrationStartupReconcilerTest { CREATED.plusSeconds(30), STARTED.plusSeconds(30), "foreign-generation"); return new DurableCutoverSnapshots(foreign, "b".repeat(64)).cleanPending(); } + + private MigrationOperationSnapshot activationRollback(MigrationOperationSnapshot source) { + return new MigrationOperationSnapshot( + source.operationId(), MigrationOperationState.RUNNING, source.target(), source.applyMode(), + MigrationStage.ROLLING_BACK, 100, source.createdAt(), source.startedAt(), null, + source.verificationState(), null, MigrationRollbackOrigin.ACTIVATION_FAILURE, + 1000, false, false, false, source.targetIdentityHash(), + source.managedCandidateGeneration()); + } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupRecoverySessionTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupRecoverySessionTest.java new file mode 100644 index 0000000000..bb33b68fda --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupRecoverySessionTest.java @@ -0,0 +1,370 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.nio.file.Path; +import java.time.Instant; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class ManagedMigrationStartupRecoverySessionTest { + + private static final String OPERATION = "operation-a"; + private static final String GENERATION = "candidate-generation"; + private static final String IDENTITY = "a".repeat(64); + private static final Instant CREATED = Instant.parse("2026-08-10T08:00:00Z"); + private static final Instant STARTED = CREATED.plusSeconds(1); + + @TempDir + private Path root; + + @Test + void bindsTheExactActionableDraftOnceAndReusesTheRuntimeAcrossRetries() { + FileMigrationOperationStore store = seededAwaitingRestart(root); + ManagedMigrationStartupRecoveryRuntime runtime = mock(ManagedMigrationStartupRecoveryRuntime.class); + AtomicReference observed = new AtomicReference<>(); + when(runtime.reconcile(any())).thenAnswer(invocation -> { + observed.compareAndSet(null, invocation.getArgument(0)); + assertThat(invocation.getArgument(0)).isEqualTo(observed.get()); + return MigrationStartupReconciliation.GATED; + }); + + try (ManagedMigrationStartupRecoverySession session = + new ManagedMigrationStartupRecoverySession(root, store, runtime)) { + assertThat(session.reconcile()) + .isEqualTo(ManagedMigrationStartupRecoveryDisposition.GATED_RECOVERY); + assertThat(session.reconcile()) + .isEqualTo(ManagedMigrationStartupRecoveryDisposition.GATED_RECOVERY); + } + + assertThat(observed.get()).isEqualTo(draft()); + verify(runtime, times(2)).reconcile(observed.get()); + verify(runtime).close(); + } + + @Test + void invokesRuntimeOnlyForTheThreeManagedRestartRecoveryShapes() { + for (MigrationOperationSnapshot snapshot : new MigrationOperationSnapshot[] { + activatingSnapshot(), awaitingRestartSnapshot(), rollingBackSnapshot(), + activationRollbackSnapshot() + }) { + Path caseRoot = root.resolve(snapshot.stage().name() + '-' + snapshot.rollbackOrigin()); + FileMigrationOperationStore store = seedSnapshot(caseRoot, snapshot); + ManagedMigrationStartupRecoveryRuntime runtime = mock(ManagedMigrationStartupRecoveryRuntime.class); + when(runtime.reconcile(any())).thenReturn(MigrationStartupReconciliation.GATED); + + try (ManagedMigrationStartupRecoverySession session = + new ManagedMigrationStartupRecoverySession(caseRoot, store, runtime)) { + assertThat(session.reconcile()) + .isEqualTo(ManagedMigrationStartupRecoveryDisposition.GATED_RECOVERY); + } + + verify(runtime).reconcile(draft()); + } + } + + @Test + void verificationFailureRollbackRemainsGatedWithoutRuntimeWork() { + MigrationOperationSnapshot invalid = rollbackSnapshot(MigrationRollbackOrigin.VERIFICATION_FAILURE); + FileMigrationOperationStore store = mock(FileMigrationOperationStore.class); + when(store.selectUniqueNonterminalForStartup()).thenReturn(Optional.of(invalid)); + ManagedMigrationStartupRecoveryRuntime runtime = mock(ManagedMigrationStartupRecoveryRuntime.class); + + try (ManagedMigrationStartupRecoverySession session = + new ManagedMigrationStartupRecoverySession(root, store, runtime)) { + assertThat(session.reconcile()) + .isEqualTo(ManagedMigrationStartupRecoveryDisposition.GATED_RECOVERY); + } + + verify(runtime, never()).reconcile(any()); + } + + @Test + void cachesManagedConvergenceAsFullGatedReloadWithoutFurtherRuntimeWork() { + FileMigrationOperationStore store = seededAwaitingRestart(root); + ManagedMigrationStartupRecoveryRuntime runtime = mock(ManagedMigrationStartupRecoveryRuntime.class); + when(runtime.reconcile(any())) + .thenReturn(MigrationStartupReconciliation.SUCCEEDED, + MigrationStartupReconciliation.NO_MIGRATION); + + try (ManagedMigrationStartupRecoverySession session = + new ManagedMigrationStartupRecoverySession(root, store, runtime)) { + assertThat(session.reconcile()) + .isEqualTo(ManagedMigrationStartupRecoveryDisposition.RELOAD_FULL_GATED); + assertThat(session.reconcile()) + .isEqualTo(ManagedMigrationStartupRecoveryDisposition.RELOAD_FULL_GATED); + } + + verify(runtime, times(1)).reconcile(any()); + } + + @Test + void boundRecordDisappearanceBeforeTerminalConvergenceRemainsGated() { + FileMigrationOperationStore store = seededAwaitingRestart(root); + ManagedMigrationStartupRecoveryRuntime runtime = mock(ManagedMigrationStartupRecoveryRuntime.class); + when(runtime.reconcile(any())).thenReturn(MigrationStartupReconciliation.NO_MIGRATION); + + try (ManagedMigrationStartupRecoverySession session = + new ManagedMigrationStartupRecoverySession(root, store, runtime)) { + assertThat(session.reconcile()) + .isEqualTo(ManagedMigrationStartupRecoveryDisposition.GATED_RECOVERY); + assertThat(session.reconcile()) + .isEqualTo(ManagedMigrationStartupRecoveryDisposition.GATED_RECOVERY); + } + + verify(runtime, times(2)).reconcile(any()); + } + + @Test + void terminalHistoryIsPermanentlyNoMigrationForThisSession() { + FileMigrationOperationStore store = seededAwaitingRestart(root); + MigrationOperationSnapshot awaiting = store.find(OPERATION).orElseThrow(); + store.compareAndTransition(OPERATION, MigrationOperationState.AWAITING_RESTART, + new MigrationStartupSnapshots(awaiting).succeeded(CREATED.plusSeconds(10))); + ManagedMigrationStartupRecoveryRuntime runtime = mock(ManagedMigrationStartupRecoveryRuntime.class); + + try (ManagedMigrationStartupRecoverySession session = + new ManagedMigrationStartupRecoverySession(root, store, runtime)) { + assertThat(session.reconcile()) + .isEqualTo(ManagedMigrationStartupRecoveryDisposition.NO_MIGRATION); + store.create(foreignPending()); + assertThat(session.reconcile()) + .isEqualTo(ManagedMigrationStartupRecoveryDisposition.NO_MIGRATION); + } + + verify(runtime, never()).reconcile(any()); + } + + @Test + void gatesPendingCopyVerifyReadyAndExternalWithoutConstructingRuntimeWork() { + for (MigrationOperationSnapshot snapshot : new MigrationOperationSnapshot[] { + new DurableCutoverSnapshots(draft(), IDENTITY).cleanPending(), + new DurableCutoverSnapshots(draft(), IDENTITY).running(), + verifyingSnapshot(), + readySnapshot(), + externalPending() + }) { + Path caseRoot = root.resolve(snapshot.operationId() + '-' + snapshot.stage().name()); + FileMigrationOperationStore store = new FileMigrationOperationStore(caseRoot); + store.createOrConfirm(snapshot.state() == MigrationOperationState.PENDING + ? snapshot : initialFor(snapshot)); + advance(store, snapshot); + ManagedMigrationStartupRecoveryRuntime runtime = mock(ManagedMigrationStartupRecoveryRuntime.class); + + try (ManagedMigrationStartupRecoverySession session = + new ManagedMigrationStartupRecoverySession(caseRoot, store, runtime)) { + assertThat(session.reconcile()) + .isEqualTo(ManagedMigrationStartupRecoveryDisposition.GATED_RECOVERY); + assertThat(session.reconcile()) + .isEqualTo(ManagedMigrationStartupRecoveryDisposition.GATED_RECOVERY); + } + + verify(runtime, never()).reconcile(any()); + } + } + + @Test + void gatesJournalConflictAndSafeRuntimeFailuresButPreservesFatalError() { + FileMigrationOperationStore corrupt = mock(FileMigrationOperationStore.class); + when(corrupt.selectUniqueNonterminalForStartup()) + .thenThrow(new MigrationOperationStoreException(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)); + ManagedMigrationStartupRecoveryRuntime unused = mock(ManagedMigrationStartupRecoveryRuntime.class); + try (ManagedMigrationStartupRecoverySession session = + new ManagedMigrationStartupRecoverySession(root, corrupt, unused)) { + assertThat(session.reconcile()) + .isEqualTo(ManagedMigrationStartupRecoveryDisposition.GATED_RECOVERY); + } + verify(unused, never()).reconcile(any()); + + ManagedMigrationStartupRecoveryRuntime failing = mock(ManagedMigrationStartupRecoveryRuntime.class); + when(failing.reconcile(any())) + .thenThrow(new MigrationStartupReconciliationException( + SetupErrorCode.CONFIG_RECOVERY_REQUIRED)); + try (ManagedMigrationStartupRecoverySession session = new ManagedMigrationStartupRecoverySession( + root.resolve("failure"), seededAwaitingRestart(root.resolve("failure")), failing)) { + assertThat(session.reconcile()) + .isEqualTo(ManagedMigrationStartupRecoveryDisposition.GATED_RECOVERY); + } + + AssertionError fatal = new AssertionError("private fatal"); + ManagedMigrationStartupRecoveryRuntime fatalRuntime = mock(ManagedMigrationStartupRecoveryRuntime.class); + when(fatalRuntime.reconcile(any())).thenThrow(fatal); + try (ManagedMigrationStartupRecoverySession session = new ManagedMigrationStartupRecoverySession( + root.resolve("fatal"), seededAwaitingRestart(root.resolve("fatal")), fatalRuntime)) { + assertThatThrownBy(session::reconcile).isSameAs(fatal); + } + } + + @Test + void closeRetriesRuntimeCleanupAndPreservesFatal() { + ManagedMigrationStartupRecoveryRuntime runtime = mock(ManagedMigrationStartupRecoveryRuntime.class); + AssertionError fatal = new AssertionError("private cleanup fatal"); + org.mockito.Mockito.doThrow(fatal).doNothing().when(runtime).close(); + ManagedMigrationStartupRecoverySession session = + new ManagedMigrationStartupRecoverySession(root, new FileMigrationOperationStore(root), runtime); + + assertThatThrownBy(session::close).isSameAs(fatal); + session.close(); + session.close(); + + verify(runtime, times(2)).close(); + } + + @Test + void publicDispositionAndSessionDoNotExposeCredentialsOrJdbcHandles() { + assertThat(ManagedMigrationStartupRecoveryDisposition.values()) + .extracting(Enum::name) + .containsExactly("NO_MIGRATION", "GATED_RECOVERY", "RELOAD_FULL_GATED"); + assertThat(ManagedMigrationStartupRecoverySession.class.getDeclaredFields()) + .extracting(field -> field.getType().getName()) + .noneMatch(name -> name.contains("SecretValue") + || name.contains("Connection") + || name.contains("DataSource")); + } + + private static FileMigrationOperationStore seededAwaitingRestart(Path root) { + FileMigrationOperationStore store = seededReady(root); + MigrationOperationSnapshot ready = store.find(OPERATION).orElseThrow(); + MigrationOperationSnapshot activating = new RetainedManagedActivationSnapshots(ready).activatingSnapshot(); + store.compareAndTransition(OPERATION, MigrationOperationState.READY_TO_ACTIVATE, activating); + store.compareAndTransition(OPERATION, MigrationOperationState.RUNNING, + new RetainedManagedActivationSnapshots(activating).awaitingRestartSnapshot()); + return store; + } + + private static FileMigrationOperationStore seededReady(Path root) { + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + MigrationOperationSnapshot ready = readySnapshot(); + MigrationOperationSnapshot pending = new DurableCutoverSnapshots(draft(), IDENTITY).cleanPending(); + store.createOrConfirm(pending); + store.compareAndTransition(OPERATION, MigrationOperationState.PENDING, + new DurableCutoverSnapshots(draft(), IDENTITY).running()); + MigrationOperationSnapshot running = store.find(OPERATION).orElseThrow(); + store.compareAndTransition(OPERATION, MigrationOperationState.RUNNING, + new RetainedCopyJournalSnapshots(running).verifyingSnapshot()); + store.compareAndTransition(OPERATION, MigrationOperationState.RUNNING, ready); + return store; + } + + private static FileMigrationOperationStore seedSnapshot( + Path root, MigrationOperationSnapshot target) { + if (target.stage() == MigrationStage.ACTIVATING) { + FileMigrationOperationStore store = seededReady(root); + store.compareAndTransition(OPERATION, MigrationOperationState.READY_TO_ACTIVATE, target); + return store; + } + if (target.stage() == MigrationStage.ROLLING_BACK + && target.rollbackOrigin() == MigrationRollbackOrigin.ACTIVATION_FAILURE) { + FileMigrationOperationStore store = seededReady(root); + store.compareAndTransition(OPERATION, MigrationOperationState.READY_TO_ACTIVATE, target); + return store; + } + FileMigrationOperationStore store = seededAwaitingRestart(root); + if (target.stage() == MigrationStage.ROLLING_BACK) { + store.compareAndTransition(OPERATION, MigrationOperationState.AWAITING_RESTART, target); + } + return store; + } + + private static MigrationOperationSnapshot activatingSnapshot() { + return new RetainedManagedActivationSnapshots(readySnapshot()).activatingSnapshot(); + } + + private static MigrationOperationSnapshot awaitingRestartSnapshot() { + return new RetainedManagedActivationSnapshots(activatingSnapshot()).awaitingRestartSnapshot(); + } + + private static MigrationOperationSnapshot rollingBackSnapshot() { + return new MigrationStartupSnapshots(awaitingRestartSnapshot()).rollingBack(); + } + + private static MigrationOperationSnapshot activationRollbackSnapshot() { + return rollbackSnapshot(MigrationRollbackOrigin.ACTIVATION_FAILURE); + } + + private static MigrationOperationSnapshot rollbackSnapshot(MigrationRollbackOrigin origin) { + MigrationOperationSnapshot source = readySnapshot(); + return new MigrationOperationSnapshot( + source.operationId(), MigrationOperationState.RUNNING, source.target(), source.applyMode(), + MigrationStage.ROLLING_BACK, 100, source.createdAt(), source.startedAt(), null, + origin.verificationState(), null, origin, 1000, false, false, false, + source.targetIdentityHash(), source.managedCandidateGeneration()); + } + + private static MigrationOperationSnapshot readySnapshot() { + MigrationOperationSnapshot running = new DurableCutoverSnapshots(draft(), IDENTITY).running(); + return new RetainedCopyJournalSnapshots(running).finalSnapshot(); + } + + private static MigrationOperationSnapshot initialFor(MigrationOperationSnapshot snapshot) { + return snapshot.applyMode() == ApplyMode.EXTERNAL_APPLY + ? externalPending() : new DurableCutoverSnapshots(draft(), IDENTITY).cleanPending(); + } + + private static void advance(FileMigrationOperationStore store, MigrationOperationSnapshot target) { + MigrationOperationSnapshot current = store.find(target.operationId()).orElseThrow(); + if (current.equals(target)) { + return; + } + if (target.state() == MigrationOperationState.RUNNING) { + MigrationOperationSnapshot running = new DurableCutoverSnapshots(draft(), IDENTITY).running(); + store.compareAndTransition(target.operationId(), MigrationOperationState.PENDING, running); + if (!running.equals(target)) { + store.compareAndTransition(target.operationId(), MigrationOperationState.RUNNING, target); + } + return; + } + if (target.state() == MigrationOperationState.READY_TO_ACTIVATE) { + MigrationOperationSnapshot running = new DurableCutoverSnapshots(draft(), IDENTITY).running(); + store.compareAndTransition(target.operationId(), MigrationOperationState.PENDING, running); + MigrationOperationSnapshot verifying = new RetainedCopyJournalSnapshots(running).verifyingSnapshot(); + store.compareAndTransition(target.operationId(), MigrationOperationState.RUNNING, verifying); + store.compareAndTransition(target.operationId(), MigrationOperationState.RUNNING, target); + } + } + + private static MigrationOperationSnapshot verifyingSnapshot() { + MigrationOperationSnapshot running = new DurableCutoverSnapshots(draft(), IDENTITY).running(); + return new RetainedCopyJournalSnapshots(running).verifyingSnapshot(); + } + + private static DurableCutoverDraft draft() { + return new DurableCutoverDraft(OPERATION, MigrationTarget.POSTGRESQL, + ApplyMode.MANAGED_WRITE, CREATED, STARTED, GENERATION); + } + + private static MigrationOperationSnapshot externalPending() { + DurableCutoverDraft external = new DurableCutoverDraft( + "operation-external", MigrationTarget.MYSQL, ApplyMode.EXTERNAL_APPLY, + CREATED, STARTED, null); + return new DurableCutoverSnapshots(external, "b".repeat(64)).cleanPending(); + } + + private static MigrationOperationSnapshot foreignPending() { + DurableCutoverDraft foreign = new DurableCutoverDraft( + "operation-b", MigrationTarget.MYSQL, ApplyMode.MANAGED_WRITE, + CREATED.plusSeconds(20), STARTED.plusSeconds(20), "foreign-generation"); + return new DurableCutoverSnapshots(foreign, "c".repeat(64)).cleanPending(); + } +} From 27633afb265a947945e34c986492ff5a440369d5 Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 17:59:22 +0800 Subject: [PATCH 62/71] Guard managed migration startup admission --- .../ManagedMigrationStartupAdmission.java | 29 ++ .../ManagedMigrationStartupAdmissionTest.java | 52 +++ ...ManagedConfigEnvironmentPostProcessor.java | 111 ++++- ...edMigrationDatasourceOverrideDetector.java | 91 ++++ .../AdmittedStartupContextLauncher.java | 24 + .../runtime/HertzBeatStartupCoordinator.java | 11 + .../runtime/SpringStartupContextLauncher.java | 58 ++- .../runtime/StartupLaunchAdmission.java | 77 ++++ .../HertzBeatStartupCoordinatorTest.java | 15 +- .../ManagedMigrationSpringAdmissionTest.java | 435 ++++++++++++++++++ .../StartupRuntimeBoundaryContextTest.java | 8 +- 11 files changed, 877 insertions(+), 34 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupAdmission.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupAdmissionTest.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/config/ManagedMigrationDatasourceOverrideDetector.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/AdmittedStartupContextLauncher.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupLaunchAdmission.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/ManagedMigrationSpringAdmissionTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupAdmission.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupAdmission.java new file mode 100644 index 0000000000..78cdcf995f --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupAdmission.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.nio.file.Path; +import java.util.Objects; + +/** Secret-free, locked pre-Spring admission for the migration operation collection. */ +public enum ManagedMigrationStartupAdmission { + CLEAR, + GATED_RECOVERY; + + /** Returns clear only when the exact locked collection has no nonterminal operation. */ + public static ManagedMigrationStartupAdmission inspect(Path installationRoot) { + Objects.requireNonNull(installationRoot, "installationRoot"); + try { + return new FileMigrationOperationStore(installationRoot) + .selectUniqueNonterminalForStartup() + .isEmpty() ? CLEAR : GATED_RECOVERY; + } catch (RuntimeException failure) { + return GATED_RECOVERY; + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupAdmissionTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupAdmissionTest.java new file mode 100644 index 0000000000..16979497cc --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationStartupAdmissionTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class ManagedMigrationStartupAdmissionTest { + + @TempDir + private Path root; + + @Test + void reportsClearOnlyWhenTheLockedCollectionHasNoNonterminalOperation() { + assertThat(ManagedMigrationStartupAdmission.inspect(root)) + .isEqualTo(ManagedMigrationStartupAdmission.CLEAR); + DurableCutoverDraft draft = new DurableCutoverDraft( + "operation-a", MigrationTarget.POSTGRESQL, ApplyMode.MANAGED_WRITE, + Instant.parse("2026-08-10T09:00:00Z"), Instant.parse("2026-08-10T09:00:01Z"), + "candidate-generation"); + new FileMigrationOperationStore(root).create( + new DurableCutoverSnapshots(draft, "a".repeat(64)).cleanPending()); + + assertThat(ManagedMigrationStartupAdmission.inspect(root)) + .isEqualTo(ManagedMigrationStartupAdmission.GATED_RECOVERY); + } + + @Test + void corruptCollectionFailsClosedWithoutExposingItsContent() throws Exception { + Path operationFile = root.resolve(FileMigrationOperationStore.RELATIVE_PATH); + Files.createDirectories(operationFile.getParent()); + Files.writeString(operationFile, "private-jdbc-password"); + + assertThat(ManagedMigrationStartupAdmission.inspect(root)) + .isEqualTo(ManagedMigrationStartupAdmission.GATED_RECOVERY); + assertThat(ManagedMigrationStartupAdmission.values()) + .extracting(Enum::name) + .containsExactly("CLEAR", "GATED_RECOVERY"); + } +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/config/ManagedConfigEnvironmentPostProcessor.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/config/ManagedConfigEnvironmentPostProcessor.java index 57e6e5c008..c62af89a19 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/config/ManagedConfigEnvironmentPostProcessor.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/config/ManagedConfigEnvironmentPostProcessor.java @@ -17,13 +17,18 @@ package org.apache.hertzbeat.startup.config; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import org.apache.hertzbeat.bootstrap.SetupOnlyApplication; import org.apache.hertzbeat.common.runtime.RuntimeMode; import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector; import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector.Inspection; import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector.State; import org.apache.hertzbeat.manager.setup.config.SetupInstallationPaths; +import org.apache.hertzbeat.manager.setup.workflow.ManagedMigrationStartupAdmission; +import org.apache.hertzbeat.startup.HertzBeatApplication; +import org.apache.hertzbeat.startup.runtime.StartupLaunchAdmission; import org.springframework.boot.EnvironmentPostProcessor; import org.springframework.boot.SpringApplication; import org.springframework.boot.context.config.ConfigDataEnvironmentPostProcessor; @@ -44,7 +49,7 @@ import org.springframework.core.io.Resource; public final class ManagedConfigEnvironmentPostProcessor implements EnvironmentPostProcessor, Ordered { static final String INSTALLATION_ROOT_PROPERTY = SetupInstallationPaths.ROOT_PROPERTY; - public static final String INTERNAL_RUNTIME_PROPERTY_SOURCE = "hertzbeatInternalRuntimeMode"; + public static final String INTERNAL_RUNTIME_PROPERTY_SOURCE = StartupLaunchAdmission.INTERNAL_PROPERTY_SOURCE; private static final String DEFAULT_INSTALLATION_ROOT = "."; @Override @@ -57,21 +62,37 @@ public final class ManagedConfigEnvironmentPostProcessor implements EnvironmentP PropertySource internalMode = environment.getPropertySources().get(INTERNAL_RUNTIME_PROPERTY_SOURCE); RuntimeMode mode = internalMode == null ? RuntimeMode.NORMAL : RuntimeMode.fromProperty((String) internalMode.getProperty(RuntimeMode.PROPERTY_NAME)); - if (internalMode != null) { - environment.getPropertySources().remove(INTERNAL_RUNTIME_PROPERTY_SOURCE); - environment.getPropertySources().addFirst(internalMode); - } if (mode == RuntimeMode.SETUP_ONLY || mode == RuntimeMode.RECOVERY) { + if (!setupOnlyApplication(application)) { + throw migrationAdmissionFailed(); + } + Path installationRoot = StartupLaunchAdmission.isTrusted(internalMode) + ? resolveInstallationRoot(environment) : null; + promoteInternalMode(environment, internalMode); + if (StartupLaunchAdmission.isTrusted(internalMode)) { + requireTrustedInstallationRoot(internalMode, installationRoot); + sanitizeStartupAdmission(environment, mode, installationRoot); + } return; } - Path installationRoot = Path.of(environment.getProperty( - INSTALLATION_ROOT_PROPERTY, DEFAULT_INSTALLATION_ROOT)).toAbsolutePath().normalize(); + if (internalMode == null && setupOnlyApplication(application)) { + return; + } + Path installationRoot = resolveInstallationRoot(environment); + promoteInternalMode(environment, internalMode); + if (StartupLaunchAdmission.isTrusted(internalMode)) { + requireTrustedInstallationRoot(internalMode, installationRoot); + } Path directory = installationRoot.resolve("data/config"); if (Files.isSymbolicLink(installationRoot) || Files.isSymbolicLink(directory.getParent()) || Files.isSymbolicLink(directory)) { throw recoveryRequired(); } Inspection inspection = new ManagedActiveConfigurationInspector(installationRoot).inspect(); + if (fullApplication(application)) { + requireStartupAdmission(environment, internalMode, installationRoot, inspection); + sanitizeStartupAdmission(environment, mode, installationRoot); + } if (inspection.state() == State.ABSENT) { return; } @@ -86,6 +107,76 @@ public final class ManagedConfigEnvironmentPostProcessor implements EnvironmentP inspection.secretProperties())); } + private static boolean fullApplication(SpringApplication application) { + return hasExactSource(application, HertzBeatApplication.class); + } + + private static boolean setupOnlyApplication(SpringApplication application) { + return hasExactSource(application, SetupOnlyApplication.class); + } + + private static boolean hasExactSource(SpringApplication application, Class source) { + return application.getAllSources().contains(source) + || application.getAllSources().contains(source.getName()); + } + + private static void requireStartupAdmission( + ConfigurableEnvironment environment, + PropertySource internalMode, + Path installationRoot, + Inspection inspection) { + if (!StartupLaunchAdmission.isTrusted(internalMode) + || ManagedMigrationStartupAdmission.inspect(installationRoot) + != ManagedMigrationStartupAdmission.CLEAR + || StartupLaunchAdmission.exactManagedDatasourceRequired(internalMode) + && inspection.state() != State.LOADABLE + || StartupLaunchAdmission.exactManagedDatasourceRequired(internalMode) + && new ManagedMigrationDatasourceOverrideDetector() + .hasOverride(environment.getPropertySources())) { + throw migrationAdmissionFailed(); + } + } + + private static void sanitizeStartupAdmission( + ConfigurableEnvironment environment, RuntimeMode mode, Path installationRoot) { + try { + environment.getPropertySources().replace( + INTERNAL_RUNTIME_PROPERTY_SOURCE, + StartupLaunchAdmission.sanitizedPropertySource(mode, installationRoot.toRealPath())); + } catch (IOException | RuntimeException failure) { + throw migrationAdmissionFailed(); + } + } + + private static Path resolveInstallationRoot(ConfigurableEnvironment environment) { + try { + return Path.of(environment.getProperty( + INSTALLATION_ROOT_PROPERTY, DEFAULT_INSTALLATION_ROOT)).toAbsolutePath().normalize(); + } catch (RuntimeException failure) { + throw migrationAdmissionFailed(); + } + } + + private static void promoteInternalMode( + ConfigurableEnvironment environment, PropertySource internalMode) { + if (internalMode != null) { + environment.getPropertySources().remove(INTERNAL_RUNTIME_PROPERTY_SOURCE); + environment.getPropertySources().addFirst(internalMode); + } + } + + private static void requireTrustedInstallationRoot( + PropertySource internalMode, Path installationRoot) { + try { + Path canonicalRoot = installationRoot.toRealPath(); + if (!StartupLaunchAdmission.isBoundTo(internalMode, canonicalRoot)) { + throw migrationAdmissionFailed(); + } + } catch (IOException | RuntimeException failure) { + throw migrationAdmissionFailed(); + } + } + private static void addBetweenExternalAndClasspath( MutablePropertySources propertySources, PropertySource managed) { for (PropertySource source : propertySources) { @@ -97,7 +188,7 @@ public final class ManagedConfigEnvironmentPostProcessor implements EnvironmentP propertySources.addLast(managed); } - private static boolean isClasspathConfigData(PropertySource source) { + static boolean isClasspathConfigData(PropertySource source) { if (!(source instanceof EnumerablePropertySource enumerable)) { return false; } @@ -122,4 +213,8 @@ public final class ManagedConfigEnvironmentPostProcessor implements EnvironmentP private static IllegalStateException recoveryRequired() { return new IllegalStateException("Managed configuration requires recovery"); } + + private static IllegalStateException migrationAdmissionFailed() { + return new IllegalStateException("Managed migration startup admission failed"); + } } diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/config/ManagedMigrationDatasourceOverrideDetector.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/config/ManagedMigrationDatasourceOverrideDetector.java new file mode 100644 index 0000000000..edf389f94a --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/config/ManagedMigrationDatasourceOverrideDetector.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.config; + +import java.util.Locale; +import java.util.Set; +import org.apache.hertzbeat.startup.runtime.StartupLaunchAdmission; +import org.springframework.core.env.EnumerablePropertySource; +import org.springframework.core.env.MutablePropertySources; +import org.springframework.core.env.PropertySource; + +/** Detects datasource identity overrides that take precedence over classpath configuration. */ +final class ManagedMigrationDatasourceOverrideDetector { + + private static final Set IDENTITY_KEYS = Set.of( + "spring.datasource.url", + "spring.datasource.username", + "spring.datasource.password", + "spring.datasource.driver-class-name", + "spring.datasource.jndi-name", + "spring.datasource.type"); + private static final Set NORMALIZED_IDENTITY_KEYS = Set.of( + "springdatasourceurl", + "springdatasourceusername", + "springdatasourcepassword", + "springdatasourcedriverclassname", + "springdatasourcejndiname", + "springdatasourcetype"); + private static final String HIKARI_PREFIX = "springdatasourcehikari"; + + boolean hasOverride(MutablePropertySources sources) { + for (PropertySource source : sources) { + if (ManagedConfigEnvironmentPostProcessor.isClasspathConfigData(source)) { + return false; + } + if (StartupLaunchAdmission.INTERNAL_PROPERTY_SOURCE.equals(source.getName())) { + continue; + } + if (knownRestrictedSource(source)) { + continue; + } + if (hasFiniteIdentityValue(source)) { + return true; + } + if (source instanceof EnumerablePropertySource enumerable) { + if (hasIdentityKey(enumerable.getPropertyNames())) { + return true; + } + } else { + return true; + } + } + return false; + } + + private boolean hasFiniteIdentityValue(PropertySource source) { + try { + for (String key : IDENTITY_KEYS) { + if (source.getProperty(key) != null) { + return true; + } + } + return false; + } catch (RuntimeException failure) { + return true; + } + } + + private boolean hasIdentityKey(String[] propertyNames) { + for (String propertyName : propertyNames) { + String normalized = propertyName.toLowerCase(Locale.ROOT).replaceAll("[._-]", ""); + if (NORMALIZED_IDENTITY_KEYS.contains(normalized) + || normalized.startsWith(HIKARI_PREFIX)) { + return true; + } + } + return false; + } + + private boolean knownRestrictedSource(PropertySource source) { + String type = source.getClass().getName(); + return type.equals("org.springframework.boot.context.properties.source." + + "ConfigurationPropertySourcesPropertySource") + || type.equals("org.springframework.boot.env.RandomValuePropertySource"); + } +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/AdmittedStartupContextLauncher.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/AdmittedStartupContextLauncher.java new file mode 100644 index 0000000000..eb32313212 --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/AdmittedStartupContextLauncher.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.runtime; + +import java.nio.file.Path; +import org.apache.hertzbeat.manager.maintenance.StandaloneDeploymentOwnerView; +import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; + +/** Internal launch capability used only while the standalone owner is valid. */ +interface AdmittedStartupContextLauncher { + + RunningApplicationContext launchAdmitted( + StartupDecision decision, + String[] args, + SetupRuntimeTransition setupRuntimeTransition, + Path installationRoot, + StandaloneDeploymentOwnerView authorityView, + StartupLaunchAdmission.Mode admissionMode); +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java index d2ec6ef194..bb5a9a6927 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java @@ -166,7 +166,18 @@ public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition launcher.launch(decision, args.clone(), this), "startup context launcher returned null for " + decision.mode().value()); } + if (!deploymentOwner.isValid()) { + throw StandaloneDeploymentOwnerException.unavailable(); + } boolean exposeAuthority = convergenceConfirmed && decision.mode() == RuntimeMode.NORMAL; + if (launcher instanceof AdmittedStartupContextLauncher admitted) { + return Objects.requireNonNull( + admitted.launchAdmitted( + decision, args.clone(), this, installationRoot.canonicalRoot(), + exposeAuthority ? deploymentOwner.view() : null, + StartupLaunchAdmission.Mode.ORDINARY), + "startup context launcher returned null for " + decision.mode().value()); + } return Objects.requireNonNull( launcher.launch(decision, args.clone(), this, installationRoot.canonicalRoot(), exposeAuthority ? deploymentOwner.view() : null), diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/SpringStartupContextLauncher.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/SpringStartupContextLauncher.java index fba1692de5..c8a84a27a9 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/SpringStartupContextLauncher.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/SpringStartupContextLauncher.java @@ -18,27 +18,25 @@ package org.apache.hertzbeat.startup.runtime; import java.nio.file.Path; -import java.util.Map; import org.apache.hertzbeat.manager.maintenance.StandaloneDeploymentOwnerView; -import org.apache.hertzbeat.manager.setup.config.SetupInstallationPaths; import org.apache.hertzbeat.bootstrap.SetupOnlyApplication; import org.apache.hertzbeat.common.runtime.RuntimeMode; import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; import org.apache.hertzbeat.startup.HertzBeatApplication; -import org.apache.hertzbeat.startup.config.ManagedConfigEnvironmentPostProcessor; import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.core.env.MapPropertySource; import org.springframework.core.env.StandardEnvironment; /** Spring implementation with explicit AOT-visible source classes. */ -public final class SpringStartupContextLauncher implements StartupContextLauncher { +public final class SpringStartupContextLauncher + implements StartupContextLauncher, AdmittedStartupContextLauncher { @Override public RunningApplicationContext launch( StartupDecision decision, String[] args, SetupRuntimeTransition setupRuntimeTransition) { ConfigurableApplicationContext context = launchSpringContext( - decision, args, setupRuntimeTransition, null, null); + decision, args, setupRuntimeTransition, null, null, + false, StartupLaunchAdmission.Mode.ORDINARY); return new SpringRunningApplicationContext(decision.mode(), context); } @@ -50,13 +48,38 @@ public final class SpringStartupContextLauncher implements StartupContextLaunche Path installationRoot, StandaloneDeploymentOwnerView authorityView) { ConfigurableApplicationContext context = launchSpringContext( - decision, args, setupRuntimeTransition, installationRoot, authorityView); + decision, args, setupRuntimeTransition, installationRoot, authorityView, + false, StartupLaunchAdmission.Mode.ORDINARY); + return new SpringRunningApplicationContext(decision.mode(), context); + } + + @Override + public RunningApplicationContext launchAdmitted( + StartupDecision decision, + String[] args, + SetupRuntimeTransition setupRuntimeTransition, + Path installationRoot, + StandaloneDeploymentOwnerView authorityView, + StartupLaunchAdmission.Mode admissionMode) { + ConfigurableApplicationContext context = launchSpringContext( + decision, args, setupRuntimeTransition, installationRoot, authorityView, + true, admissionMode); return new SpringRunningApplicationContext(decision.mode(), context); } ConfigurableApplicationContext launchSpringContext( StartupDecision decision, String[] args, SetupRuntimeTransition setupRuntimeTransition) { - return launchSpringContext(decision, args, setupRuntimeTransition, null, null); + return launchSpringContext(decision, args, setupRuntimeTransition, null, null, + false, StartupLaunchAdmission.Mode.ORDINARY); + } + + ConfigurableApplicationContext launchAdmittedSpringContext( + StartupDecision decision, + String[] args, + SetupRuntimeTransition setupRuntimeTransition, + Path installationRoot) { + return launchSpringContext(decision, args, setupRuntimeTransition, installationRoot, null, + true, StartupLaunchAdmission.Mode.ORDINARY); } private ConfigurableApplicationContext launchSpringContext( @@ -64,11 +87,13 @@ public final class SpringStartupContextLauncher implements StartupContextLaunche String[] args, SetupRuntimeTransition setupRuntimeTransition, Path installationRoot, - StandaloneDeploymentOwnerView authorityView) { + StandaloneDeploymentOwnerView authorityView, + boolean trustedLaunch, + StartupLaunchAdmission.Mode admissionMode) { StandardEnvironment environment = new StandardEnvironment(); - environment.getPropertySources().addFirst(new MapPropertySource( - ManagedConfigEnvironmentPostProcessor.INTERNAL_RUNTIME_PROPERTY_SOURCE, - internalProperties(decision, installationRoot))); + environment.getPropertySources().addFirst(trustedLaunch + ? StartupLaunchAdmission.internalPropertySource(decision, installationRoot, admissionMode) + : StartupLaunchAdmission.runtimeModePropertySource(decision)); return new SpringApplicationBuilder(sourceFor(decision.mode())) .environment(environment) .initializers(context -> { @@ -82,15 +107,6 @@ public final class SpringStartupContextLauncher implements StartupContextLaunche .run(args); } - private Map internalProperties(StartupDecision decision, Path installationRoot) { - if (installationRoot == null) { - return Map.of(RuntimeMode.PROPERTY_NAME, decision.mode().value()); - } - return Map.of( - RuntimeMode.PROPERTY_NAME, decision.mode().value(), - SetupInstallationPaths.ROOT_PROPERTY, installationRoot.toString()); - } - static Class sourceFor(RuntimeMode mode) { return mode == RuntimeMode.NORMAL || mode == RuntimeMode.FULL_SETUP_GATED ? HertzBeatApplication.class : SetupOnlyApplication.class; diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupLaunchAdmission.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupLaunchAdmission.java new file mode 100644 index 0000000000..ac96022e64 --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupLaunchAdmission.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.runtime; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.Map; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.setup.config.SetupInstallationPaths; +import org.springframework.core.env.MapPropertySource; +import org.springframework.core.env.PropertySource; + +/** Unforgeable in-process admission attached only by the official Spring context launcher. */ +public final class StartupLaunchAdmission { + + public static final String INTERNAL_PROPERTY_SOURCE = "hertzbeatInternalRuntimeMode"; + private static final String TOKEN_PROPERTY = "hertzbeat.internal.startup-admission"; + private static final String EXACT_DATASOURCE_PROPERTY = + "hertzbeat.internal.exact-managed-datasource-required"; + private static final Object TOKEN = new Object(); + + private StartupLaunchAdmission() { + } + + static MapPropertySource internalPropertySource( + StartupDecision decision, Path installationRoot, Mode mode) { + Map properties = new LinkedHashMap<>(); + properties.put(RuntimeMode.PROPERTY_NAME, decision.mode().value()); + properties.put(SetupInstallationPaths.ROOT_PROPERTY, + installationRoot.toAbsolutePath().normalize().toString()); + properties.put(TOKEN_PROPERTY, TOKEN); + properties.put(EXACT_DATASOURCE_PROPERTY, mode == Mode.EXACT_MANAGED_DATASOURCE); + return new MapPropertySource(INTERNAL_PROPERTY_SOURCE, Map.copyOf(properties)); + } + + static MapPropertySource runtimeModePropertySource(StartupDecision decision) { + return new MapPropertySource(INTERNAL_PROPERTY_SOURCE, + Map.of(RuntimeMode.PROPERTY_NAME, decision.mode().value())); + } + + public static boolean isTrusted(PropertySource source) { + return source != null && source.getProperty(TOKEN_PROPERTY) == TOKEN; + } + + public static boolean exactManagedDatasourceRequired(PropertySource source) { + return isTrusted(source) && Boolean.TRUE.equals(source.getProperty(EXACT_DATASOURCE_PROPERTY)); + } + + public static boolean isBoundTo(PropertySource source, Path canonicalRoot) { + if (!isTrusted(source)) { + return false; + } + Object configuredRoot = source.getProperty(SetupInstallationPaths.ROOT_PROPERTY); + if (!(configuredRoot instanceof String root)) { + return false; + } + try { + return Path.of(root).toAbsolutePath().normalize().toRealPath().equals(canonicalRoot); + } catch (IOException | RuntimeException failure) { + return false; + } + } + + public static MapPropertySource sanitizedPropertySource(RuntimeMode mode, Path canonicalRoot) { + return new MapPropertySource(INTERNAL_PROPERTY_SOURCE, Map.of( + RuntimeMode.PROPERTY_NAME, mode.value(), + SetupInstallationPaths.ROOT_PROPERTY, canonicalRoot.toString())); + } + + enum Mode { ORDINARY, EXACT_MANAGED_DATASOURCE } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinatorTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinatorTest.java index 519713a7ff..e6a56e7047 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinatorTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinatorTest.java @@ -309,7 +309,8 @@ class HertzBeatStartupCoordinatorTest { } } - private static final class OfficialRecordingLauncher implements StartupContextLauncher { + private static final class OfficialRecordingLauncher + implements StartupContextLauncher, AdmittedStartupContextLauncher { private final List events = new ArrayList<>(); private final List transitions = new ArrayList<>(); @@ -328,7 +329,19 @@ class HertzBeatStartupCoordinatorTest { SetupRuntimeTransition setupRuntimeTransition, Path resolvedRoot, StandaloneDeploymentOwnerView authorityView) { + throw new AssertionError("Owned startup must use the admitted launcher capability"); + } + + @Override + public RunningApplicationContext launchAdmitted( + StartupDecision decision, + String[] args, + SetupRuntimeTransition setupRuntimeTransition, + Path resolvedRoot, + StandaloneDeploymentOwnerView authorityView, + StartupLaunchAdmission.Mode admissionMode) { assertTrue(resolvedRoot.isAbsolute()); + assertEquals(StartupLaunchAdmission.Mode.ORDINARY, admissionMode); events.add("open:" + decision.mode().value()); transitions.add(setupRuntimeTransition); views.add(authorityView); diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/ManagedMigrationSpringAdmissionTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/ManagedMigrationSpringAdmissionTest.java new file mode 100644 index 0000000000..fb630e37be --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/ManagedMigrationSpringAdmissionTest.java @@ -0,0 +1,435 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.runtime; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import org.apache.hertzbeat.bootstrap.SetupOnlyApplication; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.config.GreptimeEndpoints; +import org.apache.hertzbeat.manager.setup.config.GreptimeSettings; +import org.apache.hertzbeat.manager.setup.config.ManagedActiveConfigurationInspector; +import org.apache.hertzbeat.manager.setup.config.ManagedApplicationConfig; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigurationBundle; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.ManagedSecrets; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.apache.hertzbeat.manager.setup.config.SetupInstallationPaths; +import org.apache.hertzbeat.manager.setup.workflow.FileMigrationOperationStore; +import org.apache.hertzbeat.manager.setup.workflow.MigrationOperationSnapshot; +import org.apache.hertzbeat.startup.HertzBeatApplication; +import org.apache.hertzbeat.startup.config.ManagedConfigEnvironmentPostProcessor; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.context.properties.source.ConfigurationPropertySources; +import org.springframework.boot.env.OriginTrackedMapPropertySource; +import org.springframework.boot.origin.OriginTrackedValue; +import org.springframework.boot.origin.TextResourceOrigin; +import org.springframework.core.env.MapPropertySource; +import org.springframework.core.env.PropertySource; +import org.springframework.core.env.SimpleCommandLinePropertySource; +import org.springframework.core.env.StandardEnvironment; +import org.springframework.core.env.SystemEnvironmentPropertySource; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.FileSystemResource; + +class ManagedMigrationSpringAdmissionTest { + + @TempDir + private Path root; + + @Test + void directFullApplicationWithoutLauncherTokenFailsBeforeBeanCreation() { + StandardEnvironment environment = environment(); + + assertThatThrownBy(() -> process(environment, HertzBeatApplication.class)) + .isInstanceOf(IllegalStateException.class) + .hasNoCause() + .hasMessage("Managed migration startup admission failed"); + } + + @Test + void publicRootAwareLauncherStillCannotIssueAdmissionToken() { + SpringStartupContextLauncher launcher = new SpringStartupContextLauncher(); + StartupDecision decision = new StartupDecision(RuntimeMode.FULL_SETUP_GATED); + + assertThatThrownBy(() -> { + try (RunningApplicationContext ignored = launcher.launch( + decision, + new String[] { + "--spring.profiles.active=test", + "--spring.main.web-application-type=none", + "--spring.datasource.url=jdbc:h2:mem:public-launch;MODE=MYSQL;DB_CLOSE_DELAY=-1", + "--spring.flyway.enabled=false" + }, () -> { }, root, null)) { + throw new AssertionError("Public launch unexpectedly crossed startup admission"); + } + }).isInstanceOf(IllegalStateException.class) + .hasNoCause() + .hasMessage("Managed migration startup admission failed"); + } + + @Test + void ordinaryPropertiesCannotForgeTheLauncherToken() { + StandardEnvironment environment = environment(); + environment.getPropertySources().addFirst(new SimpleCommandLinePropertySource( + "--hertzbeat.internal.startup-admission=trusted")); + + assertThatThrownBy(() -> process(environment, HertzBeatApplication.class)) + .isInstanceOf(IllegalStateException.class) + .hasNoCause(); + } + + @Test + void setupOnlyRemainsReachableWithoutFullApplicationToken() { + assertThatCode(() -> process(environment(), SetupOnlyApplication.class)) + .doesNotThrowAnyException(); + StandardEnvironment publicLaunch = environment(); + publicLaunch.getPropertySources().addFirst(StartupLaunchAdmission.runtimeModePropertySource( + new StartupDecision(RuntimeMode.SETUP_ONLY))); + assertThatCode(() -> process(publicLaunch, SetupOnlyApplication.class)) + .doesNotThrowAnyException(); + } + + @Test + void untrustedSetupModesCannotBypassAdmissionForFullApplicationSources() { + for (RuntimeMode mode : new RuntimeMode[] { RuntimeMode.SETUP_ONLY, RuntimeMode.RECOVERY }) { + for (Object source : new Object[] { HertzBeatApplication.class, HertzBeatApplication.class.getName() }) { + StandardEnvironment environment = environment(); + environment.getPropertySources().addFirst(StartupLaunchAdmission.runtimeModePropertySource( + new StartupDecision(mode))); + + assertThatThrownBy(() -> process(environment, source)) + .isInstanceOf(IllegalStateException.class) + .hasNoCause() + .hasMessage("Managed migration startup admission failed"); + } + } + } + + @Test + void admittedSetupOnlyLaunchSanitizesCapabilityBeforeBeansCanObserveIt() throws Exception { + assertAdmittedSetupSourceSanitizedAndNotReplayable(RuntimeMode.SETUP_ONLY); + } + + @Test + void admittedRecoveryLaunchSanitizesCapabilityBeforeBeansCanObserveIt() throws Exception { + assertAdmittedSetupSourceSanitizedAndNotReplayable(RuntimeMode.RECOVERY); + } + + @Test + void setupOnlyClassAndClassNameBypassCorruptManagedPair() throws Exception { + Path config = Files.createDirectories(root.resolve("data/config")); + Files.writeString(config.resolve("managed-application.yml"), "invalid"); + + assertThatCode(() -> process(environment(), SetupOnlyApplication.class)) + .doesNotThrowAnyException(); + assertThatCode(() -> process(environment(), SetupOnlyApplication.class.getName())) + .doesNotThrowAnyException(); + } + + @Test + void fullApplicationClassNameRequiresExactLauncherAdmission() { + assertThatThrownBy(() -> process(environment(), HertzBeatApplication.class.getName())) + .isInstanceOf(IllegalStateException.class) + .hasNoCause() + .hasMessage("Managed migration startup admission failed"); + } + + @Test + void trustedFullLaunchStillFailsClosedForNonterminalMigration() { + seedPending(); + StandardEnvironment environment = trustedEnvironment( + new StartupDecision(RuntimeMode.FULL_SETUP_GATED)); + + assertThatThrownBy(() -> process(environment, HertzBeatApplication.class)) + .isInstanceOf(IllegalStateException.class) + .hasNoCause(); + } + + @Test + void trustedFullLaunchIsAdmittedWhenMigrationJournalIsClear() { + assertThatCode(() -> process( + trustedEnvironment(new StartupDecision(RuntimeMode.FULL_SETUP_GATED)), + HertzBeatApplication.class)).doesNotThrowAnyException(); + } + + @Test + void admittedPropertySourceCannotBeReplayedForAnotherInstallationRoot() throws Exception { + Path firstRoot = Files.createDirectories(root.resolve("first")); + Path secondRoot = root.resolve("second"); + StandardEnvironment admitted = trustedEnvironment( + new StartupDecision(RuntimeMode.FULL_SETUP_GATED), firstRoot, + StartupLaunchAdmission.Mode.ORDINARY); + process(admitted, HertzBeatApplication.class); + MapPropertySource exposed = (MapPropertySource) admitted.getPropertySources() + .get(StartupLaunchAdmission.INTERNAL_PROPERTY_SOURCE); + Map replay = new LinkedHashMap<>(exposed.getSource()); + replay.put(SetupInstallationPaths.ROOT_PROPERTY, secondRoot.toString()); + StandardEnvironment direct = environment(secondRoot); + direct.getPropertySources().addFirst( + new MapPropertySource(StartupLaunchAdmission.INTERNAL_PROPERTY_SOURCE, replay)); + + assertThatThrownBy(() -> process(direct, HertzBeatApplication.class)) + .isInstanceOf(IllegalStateException.class) + .hasNoCause() + .hasMessage("Managed migration startup admission failed"); + } + + @Test + void launcherAdmissionCannotBeRedirectedToHigherPrecedenceInstallationRoot() throws Exception { + Path issuedRoot = Files.createDirectories(root.resolve("issued")); + Path redirectedRoot = Files.createDirectories(root.resolve("redirected")); + StandardEnvironment environment = trustedEnvironment( + new StartupDecision(RuntimeMode.FULL_SETUP_GATED), issuedRoot, + StartupLaunchAdmission.Mode.ORDINARY); + environment.getPropertySources().addBefore( + StartupLaunchAdmission.INTERNAL_PROPERTY_SOURCE, + new MapPropertySource("higher-precedence-root", + Map.of(SetupInstallationPaths.ROOT_PROPERTY, redirectedRoot.toString()))); + + assertThatThrownBy(() -> process(environment, HertzBeatApplication.class)) + .isInstanceOf(IllegalStateException.class) + .hasNoCause() + .hasMessage("Managed migration startup admission failed"); + } + + @Test + void exactManagedReloadRequiresLoadableManagedConfiguration() { + StandardEnvironment environment = trustedEnvironment( + new StartupDecision(RuntimeMode.FULL_SETUP_GATED), + StartupLaunchAdmission.Mode.EXACT_MANAGED_DATASOURCE); + + assertRejected(environment); + } + + @Test + void successfulAdmissionSanitizesCapabilityAndStillInsertsManagedSources() throws Exception { + applyManagedConfiguration(); + StandardEnvironment environment = trustedEnvironment( + new StartupDecision(RuntimeMode.FULL_SETUP_GATED), + StartupLaunchAdmission.Mode.EXACT_MANAGED_DATASOURCE); + + process(environment, HertzBeatApplication.class); + + MapPropertySource sanitized = (MapPropertySource) environment.getPropertySources() + .get(StartupLaunchAdmission.INTERNAL_PROPERTY_SOURCE); + assertThat(sanitized.getSource()).containsOnlyKeys( + RuntimeMode.PROPERTY_NAME, SetupInstallationPaths.ROOT_PROPERTY); + assertThat(environment.getPropertySources() + .get(ManagedActiveConfigurationInspector.MANAGED_APPLICATION_SOURCE)).isNotNull(); + assertThat(environment.getPropertySources() + .get(ManagedActiveConfigurationInspector.MANAGED_SECRET_SOURCE)).isNotNull(); + } + + @Test + void exactManagedReloadIgnoresAttachedAggregateViewOfClasspathDatasource() throws Exception { + applyManagedConfiguration(); + StandardEnvironment environment = trustedEnvironment( + new StartupDecision(RuntimeMode.FULL_SETUP_GATED), + StartupLaunchAdmission.Mode.EXACT_MANAGED_DATASOURCE); + TextResourceOrigin origin = new TextResourceOrigin( + new ClassPathResource("application.yml"), + new TextResourceOrigin.Location(1, 1)); + environment.getPropertySources().addLast(new OriginTrackedMapPropertySource( + "Config resource 'class path resource [application.yml]'", + Map.of("spring.datasource.url", OriginTrackedValue.of("classpath-default", origin)))); + ConfigurationPropertySources.attach(environment); + + assertThatCode(() -> process(environment, HertzBeatApplication.class)) + .doesNotThrowAnyException(); + } + + @Test + void exactManagedReloadRejectsDatasourceIdentityOverridesFromEveryExternalSource() throws Exception { + applyManagedConfiguration(); + for (String property : new String[] { + "spring.datasource.url", "spring.datasource.username", "spring.datasource.password", + "spring.datasource.driver-class-name", "spring.datasource.jndi-name", "spring.datasource.type", + "spring.datasource.hikari.data-source-properties.socketFactory", + "spring.datasource.hikari.jdbc-url" + }) { + StandardEnvironment commandLine = trustedEnvironment( + new StartupDecision(RuntimeMode.FULL_SETUP_GATED), + StartupLaunchAdmission.Mode.EXACT_MANAGED_DATASOURCE); + commandLine.getPropertySources().addAfter( + StartupLaunchAdmission.INTERNAL_PROPERTY_SOURCE, + new SimpleCommandLinePropertySource("--" + property + "=private-value")); + assertRejected(commandLine); + } + + StandardEnvironment system = trustedEnvironment( + new StartupDecision(RuntimeMode.FULL_SETUP_GATED), + StartupLaunchAdmission.Mode.EXACT_MANAGED_DATASOURCE); + system.getPropertySources().replace(StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME, + new MapPropertySource(StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME, + Map.of("spring.datasource.url", "private-value"))); + assertRejected(system); + + StandardEnvironment environment = trustedEnvironment( + new StartupDecision(RuntimeMode.FULL_SETUP_GATED), + StartupLaunchAdmission.Mode.EXACT_MANAGED_DATASOURCE); + environment.getPropertySources().replace(StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME, + new SystemEnvironmentPropertySource(StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME, + new LinkedHashMap<>(Map.of("SPRING_DATASOURCE_PASSWORD", "private-value")))); + assertRejected(environment); + + StandardEnvironment externalFile = trustedEnvironment( + new StartupDecision(RuntimeMode.FULL_SETUP_GATED), + StartupLaunchAdmission.Mode.EXACT_MANAGED_DATASOURCE); + TextResourceOrigin origin = new TextResourceOrigin( + new FileSystemResource(root.resolve("external-application.yml")), + new TextResourceOrigin.Location(1, 1)); + externalFile.getPropertySources().addAfter( + StartupLaunchAdmission.INTERNAL_PROPERTY_SOURCE, + new OriginTrackedMapPropertySource("Config resource 'external-application.yml'", + Map.of("spring.datasource.url", OriginTrackedValue.of("private-value", origin)))); + assertRejected(externalFile); + } + + @Test + void exactManagedReloadFailsClosedForNonEnumerableHigherPrecedenceSources() throws Exception { + applyManagedConfiguration(); + StandardEnvironment finiteIdentity = trustedEnvironment( + new StartupDecision(RuntimeMode.FULL_SETUP_GATED), + StartupLaunchAdmission.Mode.EXACT_MANAGED_DATASOURCE); + finiteIdentity.getPropertySources().addAfter( + StartupLaunchAdmission.INTERNAL_PROPERTY_SOURCE, + nonEnumerable("finite-identity", "spring.datasource.password")); + assertRejected(finiteIdentity); + + StandardEnvironment unknownKeys = trustedEnvironment( + new StartupDecision(RuntimeMode.FULL_SETUP_GATED), + StartupLaunchAdmission.Mode.EXACT_MANAGED_DATASOURCE); + unknownKeys.getPropertySources().addAfter( + StartupLaunchAdmission.INTERNAL_PROPERTY_SOURCE, + nonEnumerable("unknown-keys", null)); + assertRejected(unknownKeys); + } + + private void assertRejected(StandardEnvironment environment) { + assertThatThrownBy(() -> process(environment, HertzBeatApplication.class)) + .isInstanceOf(IllegalStateException.class) + .hasNoCause() + .hasMessage("Managed migration startup admission failed") + .hasMessageNotContaining("private-value"); + } + + private void assertAdmittedSetupSourceSanitizedAndNotReplayable(RuntimeMode mode) throws Exception { + Path firstRoot = Files.createDirectories(root.resolve(mode.value()).resolve("first")); + Path secondRoot = root.resolve(mode.value()).resolve("second"); + StandardEnvironment admitted = trustedEnvironment( + new StartupDecision(mode), firstRoot, StartupLaunchAdmission.Mode.ORDINARY); + + process(admitted, SetupOnlyApplication.class); + + MapPropertySource sanitized = (MapPropertySource) admitted.getPropertySources() + .get(StartupLaunchAdmission.INTERNAL_PROPERTY_SOURCE); + assertThat(sanitized.getSource()).containsOnlyKeys( + RuntimeMode.PROPERTY_NAME, SetupInstallationPaths.ROOT_PROPERTY); + Map replay = new LinkedHashMap<>(sanitized.getSource()); + replay.put(SetupInstallationPaths.ROOT_PROPERTY, secondRoot.toString()); + StandardEnvironment direct = environment(secondRoot); + direct.getPropertySources().addFirst( + new MapPropertySource(StartupLaunchAdmission.INTERNAL_PROPERTY_SOURCE, replay)); + assertThatThrownBy(() -> process(direct, HertzBeatApplication.class)) + .isInstanceOf(IllegalStateException.class) + .hasNoCause() + .hasMessage("Managed migration startup admission failed"); + } + + private StandardEnvironment trustedEnvironment(StartupDecision decision) { + return trustedEnvironment(decision, StartupLaunchAdmission.Mode.ORDINARY); + } + + private StandardEnvironment trustedEnvironment( + StartupDecision decision, StartupLaunchAdmission.Mode mode) { + return trustedEnvironment(decision, root, mode); + } + + private StandardEnvironment trustedEnvironment( + StartupDecision decision, Path installationRoot, StartupLaunchAdmission.Mode mode) { + StandardEnvironment environment = environment(installationRoot); + environment.getPropertySources().addFirst( + StartupLaunchAdmission.internalPropertySource(decision, installationRoot, mode)); + return environment; + } + + private StandardEnvironment environment() { + return environment(root); + } + + private StandardEnvironment environment(Path installationRoot) { + StandardEnvironment environment = new StandardEnvironment(); + environment.getPropertySources().addFirst(new MapPropertySource( + "test-installation-root", + Map.of(SetupInstallationPaths.ROOT_PROPERTY, installationRoot.toString()))); + return environment; + } + + private PropertySource nonEnumerable(String name, String visibleKey) { + return new PropertySource<>(name, new Object()) { + @Override + public Object getProperty(String name) { + return name.equals(visibleKey) ? "private-value" : null; + } + }; + } + + private void applyManagedConfiguration() throws IOException { + ManagedApplicationConfig application = new ManagedApplicationConfig( + new MetadataDatabaseSettings( + MetadataDatabaseKind.H2, "jdbc:h2:file:./data/hertzbeat", "sa"), + GreptimeSettings.anonymous( + new GreptimeEndpoints("greptime:4001", "http://greptime:4000"), "public")); + try (ManagedSecrets secrets = ManagedSecrets.withoutTelemetryPassword( + SecretValue.of("managed-password"))) { + new ManagedConfigurationTransaction(root).apply( + new ManagedConfigurationBundle(application, secrets)); + } + } + + private void process(StandardEnvironment environment, Object source) { + SpringApplication application; + if (source instanceof Class sourceClass) { + application = new SpringApplication(sourceClass); + } else { + application = new SpringApplication(); + application.setSources(Set.of((String) source)); + } + new ManagedConfigEnvironmentPostProcessor().postProcessEnvironment( + environment, application); + } + + private void seedPending() { + new FileMigrationOperationStore(root).create( + new MigrationOperationSnapshot( + "operation-a", MigrationOperationState.PENDING, MigrationTarget.POSTGRESQL, + ApplyMode.MANAGED_WRITE, MigrationStage.QUEUED, 0, + Instant.parse("2026-08-10T09:00:00Z"), null, null, + VerificationState.PENDING, null, null, 1000, + false, false, false, "a".repeat(64), "candidate-generation")); + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupRuntimeBoundaryContextTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupRuntimeBoundaryContextTest.java index 34b5871986..1f7d22b414 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupRuntimeBoundaryContextTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupRuntimeBoundaryContextTest.java @@ -106,7 +106,7 @@ class StartupRuntimeBoundaryContextTest { SpringStartupContextLauncher launcher = new SpringStartupContextLauncher(); StartupDecision decision = new StartupDecision(RuntimeMode.FULL_SETUP_GATED); String databaseName = "m5_setup_security_" + System.nanoTime(); - try (ConfigurableApplicationContext context = launcher.launchSpringContext(decision, new String[]{ + try (ConfigurableApplicationContext context = launcher.launchAdmittedSpringContext(decision, new String[]{ "--spring.profiles.active=test", "--server.port=0", "--spring.datasource.url=jdbc:h2:mem:" + databaseName + ";MODE=MYSQL;DB_CLOSE_DELAY=-1", @@ -115,7 +115,7 @@ class StartupRuntimeBoundaryContextTest { "--warehouse.store.duckdb.enabled=false", "--warehouse.store.greptime.enabled=false", "--hertzbeat.runtime.mode=normal" - }, SETUP_RUNTIME_TRANSITION); + }, SETUP_RUNTIME_TRANSITION, installationRoot); HttpClient client = HttpClient.newHttpClient()) { int port = ((WebServerApplicationContext) context).getWebServer().getPort(); @@ -148,7 +148,7 @@ class StartupRuntimeBoundaryContextTest { SpringStartupContextLauncher launcher = new SpringStartupContextLauncher(); StartupDecision decision = new StartupDecision(RuntimeMode.FULL_SETUP_GATED); String databaseName = "m2_gated_" + System.nanoTime(); - try (ConfigurableApplicationContext context = launcher.launchSpringContext(decision, new String[]{ + try (ConfigurableApplicationContext context = launcher.launchAdmittedSpringContext(decision, new String[]{ "--spring.profiles.active=test", "--spring.main.web-application-type=none", "--spring.datasource.url=jdbc:h2:mem:" + databaseName + ";MODE=MYSQL;DB_CLOSE_DELAY=-1", @@ -163,7 +163,7 @@ class StartupRuntimeBoundaryContextTest { "--warehouse.store.victoria-metrics.cluster.enabled=true", "--warehouse.store.questdb.enabled=true", "--hertzbeat.runtime.mode=normal" - }, SETUP_RUNTIME_TRANSITION)) { + }, SETUP_RUNTIME_TRANSITION, installationRoot)) { BusinessRuntimeGate gate = context.getBean(BusinessRuntimeGate.class); assertSame(SETUP_RUNTIME_TRANSITION, context.getBean(SetupRuntimeTransition.class)); assertEquals(RuntimeMode.FULL_SETUP_GATED, gate.mode()); From 5b4f18fdbc9b3326fb9f0a5b9d3b8a498768b507 Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 18:33:27 +0800 Subject: [PATCH 63/71] Reconcile migration state before Spring startup --- .../runtime/HertzBeatStartupCoordinator.java | 284 +++++++++++---- ...oundStartupMigrationRecoveryPreflight.java | 51 +++ .../startup/runtime/StartupCleanup.java | 56 +++ .../StartupMigrationAbortExecutor.java | 29 ++ .../StartupMigrationPreflightGate.java | 113 ++++++ .../StartupMigrationRecoveryPreflight.java | 19 + ...rtupMigrationRecoveryPreflightFactory.java | 34 ++ ...ertzBeatStartupMigrationLifecycleTest.java | 339 ++++++++++++++++++ ...BeatStartupMigrationPreflightFlowTest.java | 208 +++++++++++ .../StartupMigrationPreflightTestSupport.java | 185 ++++++++++ 10 files changed, 1248 insertions(+), 70 deletions(-) create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/OwnerBoundStartupMigrationRecoveryPreflight.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupCleanup.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupMigrationAbortExecutor.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupMigrationPreflightGate.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupMigrationRecoveryPreflight.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupMigrationRecoveryPreflightFactory.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupMigrationLifecycleTest.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupMigrationPreflightFlowTest.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupMigrationPreflightTestSupport.java diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java index bb5a9a6927..bd6180bf5f 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java @@ -17,28 +17,36 @@ package org.apache.hertzbeat.startup.runtime; +import java.time.Duration; import java.util.Objects; +import java.util.concurrent.Executor; import org.apache.hertzbeat.common.runtime.RuntimeMode; import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; -/** Serializes setup-to-normal transitions and always closes the old context first. */ +/** Serializes pre-Spring recovery and setup-to-normal context transitions. */ public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition, AutoCloseable { + private static final Duration MIGRATION_RECOVERY_TIMEOUT = Duration.ofSeconds(30); private final StartupDecisionProbe probe; private final StartupContextLauncher launcher; private final StartupFailureReporter failureReporter; private final StartupInstallationRootResolver rootResolver; private final StandaloneDeploymentOwnerFactory ownerFactory; + private final StartupMigrationRecoveryPreflightFactory preflightFactory; + private final Duration migrationRecoveryTimeout; + private final Executor abortExecutor; private String[] args = new String[0]; private RunningApplicationContext currentContext; private ResolvedStartupInstallationRoot installationRoot; private StandaloneDeploymentOwner deploymentOwner; + private StartupMigrationPreflightGate migrationPreflight; private boolean normalRuntimeSelected; private boolean convergenceConfirmed; + private boolean migrationCompletionApplied; private boolean closed; public HertzBeatStartupCoordinator(StartupDecisionProbe probe, StartupContextLauncher launcher) { - this(probe, launcher, new StartupFailureReporter(), null, null); + this(probe, launcher, new StartupFailureReporter(), null, null, null, null, null); } public HertzBeatStartupCoordinator( @@ -46,12 +54,14 @@ public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition StartupContextLauncher launcher, StartupInstallationRootResolver rootResolver, StandaloneDeploymentOwnerFactory ownerFactory) { - this(probe, launcher, new StartupFailureReporter(), rootResolver, ownerFactory); + this(probe, launcher, new StartupFailureReporter(), rootResolver, ownerFactory, + StartupMigrationRecoveryPreflightFactory.system(), MIGRATION_RECOVERY_TIMEOUT, + StartupMigrationAbortExecutor.processLifetime()); } HertzBeatStartupCoordinator( StartupDecisionProbe probe, StartupContextLauncher launcher, StartupFailureReporter failureReporter) { - this(probe, launcher, failureReporter, null, null); + this(probe, launcher, failureReporter, null, null, null, null, null); } HertzBeatStartupCoordinator( @@ -60,11 +70,28 @@ public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition StartupFailureReporter failureReporter, StartupInstallationRootResolver rootResolver, StandaloneDeploymentOwnerFactory ownerFactory) { + this(probe, launcher, failureReporter, rootResolver, ownerFactory, + StartupMigrationRecoveryPreflightFactory.system(), MIGRATION_RECOVERY_TIMEOUT, + StartupMigrationAbortExecutor.processLifetime()); + } + + HertzBeatStartupCoordinator( + StartupDecisionProbe probe, + StartupContextLauncher launcher, + StartupFailureReporter failureReporter, + StartupInstallationRootResolver rootResolver, + StandaloneDeploymentOwnerFactory ownerFactory, + StartupMigrationRecoveryPreflightFactory preflightFactory, + Duration migrationRecoveryTimeout, + Executor abortExecutor) { this.probe = Objects.requireNonNull(probe, "probe"); this.launcher = Objects.requireNonNull(launcher, "launcher"); this.failureReporter = Objects.requireNonNull(failureReporter, "failureReporter"); this.rootResolver = rootResolver; this.ownerFactory = ownerFactory; + this.preflightFactory = preflightFactory; + this.migrationRecoveryTimeout = migrationRecoveryTimeout; + this.abortExecutor = abortExecutor; } public synchronized RunningApplicationContext start(String[] applicationArgs) { @@ -72,77 +99,58 @@ public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition throw StandaloneDeploymentOwnerException.unavailable(); } args = applicationArgs == null ? new String[0] : applicationArgs.clone(); - acquireDeploymentOwner(); - StartupDecision decision; try { - decision = probeDecision(); - } catch (RuntimeException exception) { - failureReporter.report(StartupFailureReporter.Stage.STARTUP_PROBE, RuntimeMode.RECOVERY, exception); - decision = StartupDecision.recovery(); - } - try { - return transition(decision); - } catch (RuntimeException exception) { - releaseOwnerAfterFailedStart(); - throw exception; + acquireDeploymentOwner(); + openMigrationPreflight(); + return transitionInternal(startupPlan()); + } catch (Throwable failure) { + closed = true; + StartupCleanup.rethrow(cleanupResources(failure)); + throw new AssertionError("unreachable"); } } @Override public synchronized void configurationApplied() { - if (normalRuntimeSelected) { + if (closed) { return; } - // The intent may be stale; the durable startup probe remains authoritative for the target mode. - StartupDecision currentDecision = Objects.requireNonNull( - probeDecision(), "startup decision"); - transition(currentDecision); + if (normalRuntimeSelected && (migrationPreflight == null || migrationCompletionApplied + || !migrationPreflight.requiresReconciliationAfterNormal())) { + return; + } + StartupPlan plan = startupPlan(); + transitionInternal(plan); } @Override public synchronized void completeSetup() { + if (closed) { + return; + } if (currentContext == null || currentContext.mode() != RuntimeMode.FULL_SETUP_GATED) { return; } convergenceConfirmed = true; - transition(StartupDecision.normal()); + boolean exactMigration = migrationPreflight != null + && migrationPreflight.exactManagedDatasourceRequired(); + StartupLaunchAdmission.Mode mode = exactMigration + ? StartupLaunchAdmission.Mode.EXACT_MANAGED_DATASOURCE + : StartupLaunchAdmission.Mode.ORDINARY; + transitionInternal(new StartupPlan(StartupDecision.normal(), false, mode)); + migrationCompletionApplied = exactMigration && normalRuntimeSelected; } public synchronized RunningApplicationContext transition(StartupDecision decision) { + if (closed) { + throw StandaloneDeploymentOwnerException.unavailable(); + } Objects.requireNonNull(decision, "decision"); - if (currentContext != null && currentContext.isActive() && currentContext.mode() == decision.mode()) { - recordNormalSelection(); - return currentContext; - } - closeCurrent(); - try { - currentContext = launch(decision); - } catch (RuntimeException launchFailure) { - if (decision.mode() == RuntimeMode.RECOVERY) { - failureReporter.report(StartupFailureReporter.Stage.RECOVERY_LAUNCH, RuntimeMode.RECOVERY, launchFailure); - throw launchFailure; - } - failureReporter.report(StartupFailureReporter.Stage.CONTEXT_LAUNCH, decision.mode(), launchFailure); - try { - currentContext = launch(StartupDecision.recovery()); - } catch (RuntimeException recoveryFailure) { - failureReporter.report( - StartupFailureReporter.Stage.RECOVERY_LAUNCH, RuntimeMode.RECOVERY, recoveryFailure); - if (recoveryFailure != launchFailure) { - recoveryFailure.addSuppressed(launchFailure); - } - throw recoveryFailure; - } - } - recordNormalSelection(); - return currentContext; - } - - private void recordNormalSelection() { - if (currentContext != null && currentContext.isActive() - && currentContext.mode() == RuntimeMode.NORMAL) { - normalRuntimeSelected = true; + if (migrationPreflight != null) { + migrationPreflight.requirePublicTransitionAllowed(decision); } + return transitionInternal(new StartupPlan( + decision, false, StartupLaunchAdmission.Mode.ORDINARY)); } public synchronized RuntimeMode mode() { @@ -153,29 +161,79 @@ public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition return currentContext; } + private RunningApplicationContext transitionInternal(StartupPlan plan) { + StartupDecision decision = plan.decision(); + if (migrationPreflight != null) { + migrationPreflight.requireValidOwner(); + } + if (!plan.forceFresh() && currentContext != null && currentContext.isActive() + && currentContext.mode() == decision.mode()) { + recordNormalSelection(); + return currentContext; + } + closeCurrent(); + try { + currentContext = launch(decision, plan.admissionMode()); + } catch (RuntimeException launchFailure) { + currentContext = launchRecovery(decision, launchFailure); + } + recordNormalSelection(); + return currentContext; + } + + private RunningApplicationContext launchRecovery( + StartupDecision failedDecision, RuntimeException launchFailure) { + if (failedDecision.mode() == RuntimeMode.RECOVERY) { + failureReporter.report(StartupFailureReporter.Stage.RECOVERY_LAUNCH, + RuntimeMode.RECOVERY, launchFailure); + throw launchFailure; + } + failureReporter.report(StartupFailureReporter.Stage.CONTEXT_LAUNCH, + failedDecision.mode(), launchFailure); + try { + return launch(StartupDecision.recovery(), StartupLaunchAdmission.Mode.ORDINARY); + } catch (Error recoveryFailure) { + recoveryFailure.addSuppressed(launchFailure); + throw recoveryFailure; + } catch (RuntimeException recoveryFailure) { + failureReporter.report(StartupFailureReporter.Stage.RECOVERY_LAUNCH, + RuntimeMode.RECOVERY, recoveryFailure); + if (recoveryFailure != launchFailure) { + recoveryFailure.addSuppressed(launchFailure); + } + throw recoveryFailure; + } + } + + private void recordNormalSelection() { + if (currentContext != null && currentContext.isActive() + && currentContext.mode() == RuntimeMode.NORMAL) { + normalRuntimeSelected = true; + } + } + private void closeCurrent() { if (currentContext != null) { - currentContext.close(); + RunningApplicationContext closing = currentContext; + closing.close(); currentContext = null; } } - private RunningApplicationContext launch(StartupDecision decision) { + private RunningApplicationContext launch( + StartupDecision decision, StartupLaunchAdmission.Mode admissionMode) { if (deploymentOwner == null) { return Objects.requireNonNull( launcher.launch(decision, args.clone(), this), "startup context launcher returned null for " + decision.mode().value()); } - if (!deploymentOwner.isValid()) { - throw StandaloneDeploymentOwnerException.unavailable(); - } + requireValidOwner(); boolean exposeAuthority = convergenceConfirmed && decision.mode() == RuntimeMode.NORMAL; if (launcher instanceof AdmittedStartupContextLauncher admitted) { return Objects.requireNonNull( admitted.launchAdmitted( decision, args.clone(), this, installationRoot.canonicalRoot(), - exposeAuthority ? deploymentOwner.view() : null, - StartupLaunchAdmission.Mode.ORDINARY), + exposeAuthority ? deploymentOwner.view() : null, admissionMode), "startup context launcher returned null for " + decision.mode().value()); } return Objects.requireNonNull( @@ -184,6 +242,32 @@ public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition "startup context launcher returned null for " + decision.mode().value()); } + private StartupPlan startupPlan() { + if (migrationPreflight == null) { + return new StartupPlan(safeProbeDecision(), false, StartupLaunchAdmission.Mode.ORDINARY); + } + return switch (migrationPreflight.reconcile()) { + case PROBE -> new StartupPlan( + safeProbeDecision(), false, StartupLaunchAdmission.Mode.ORDINARY); + case GATED_RECOVERY -> new StartupPlan( + StartupDecision.recovery(), false, StartupLaunchAdmission.Mode.ORDINARY); + case RELOAD_FULL_GATED -> new StartupPlan( + new StartupDecision(RuntimeMode.FULL_SETUP_GATED), + migrationPreflight.claimFreshReload(), + StartupLaunchAdmission.Mode.EXACT_MANAGED_DATASOURCE); + }; + } + + private StartupDecision safeProbeDecision() { + try { + return probeDecision(); + } catch (RuntimeException failure) { + failureReporter.report(StartupFailureReporter.Stage.STARTUP_PROBE, + RuntimeMode.RECOVERY, failure); + return StartupDecision.recovery(); + } + } + private StartupDecision probeDecision() { StartupDecision decision = installationRoot == null ? probe.probe(args.clone()) @@ -198,25 +282,85 @@ public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition installationRoot = rootResolver.resolve(args.clone()); deploymentOwner = Objects.requireNonNull( ownerFactory.acquire(installationRoot), "standalone deployment owner"); + requireValidOwner(); } - private void releaseOwnerAfterFailedStart() { - if (currentContext == null && deploymentOwner != null) { - deploymentOwner.close(); - deploymentOwner = null; + private void openMigrationPreflight() { + if (migrationPreflight != null || deploymentOwner == null || preflightFactory == null) { + return; + } + migrationPreflight = StartupMigrationPreflightGate.open( + installationRoot.canonicalRoot(), deploymentOwner.view(), preflightFactory, + Objects.requireNonNull(migrationRecoveryTimeout, "migrationRecoveryTimeout"), + Objects.requireNonNull(abortExecutor, "abortExecutor")); + migrationPreflight.requireValidOwner(); + } + + private void requireValidOwner() { + if (migrationPreflight != null) { + migrationPreflight.requireValidOwner(); + } else if (deploymentOwner == null || !deploymentOwner.isValid()) { + throw StandaloneDeploymentOwnerException.unavailable(); } } @Override public synchronized void close() { - if (closed) { + if (closed && currentContext == null && migrationPreflight == null && deploymentOwner == null) { return; } closed = true; - closeCurrent(); - if (deploymentOwner != null) { - deploymentOwner.close(); - deploymentOwner = null; + StartupCleanup.rethrow(cleanupResources(null)); + } + + private Throwable cleanupResources(Throwable primary) { + boolean interrupted = Thread.interrupted(); + Throwable failure = primary; + try { + if (currentContext != null) { + RunningApplicationContext context = currentContext; + StartupCleanup.Result attempt = StartupCleanup.runInterruptSafe(failure, context::close); + interrupted |= attempt.interrupted(); + if (attempt.completed()) { + currentContext = null; + } + failure = attempt.failure(); + } + if (migrationPreflight != null) { + StartupMigrationPreflightGate preflight = migrationPreflight; + StartupCleanup.Result attempt = StartupCleanup.runInterruptSafe(failure, preflight::close); + interrupted |= attempt.interrupted(); + if (attempt.completed()) { + migrationPreflight = null; + } + failure = attempt.failure(); + } + if (deploymentOwner != null && currentContext == null && migrationPreflight == null) { + StandaloneDeploymentOwner owner = deploymentOwner; + StartupCleanup.Result attempt = StartupCleanup.runInterruptSafe(failure, owner::close); + interrupted |= attempt.interrupted(); + if (attempt.completed()) { + deploymentOwner = null; + } + failure = attempt.failure(); + } + return failure; + } finally { + interrupted |= Thread.interrupted(); + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + + private record StartupPlan( + StartupDecision decision, + boolean forceFresh, + StartupLaunchAdmission.Mode admissionMode) { + + private StartupPlan { + Objects.requireNonNull(decision, "decision"); + Objects.requireNonNull(admissionMode, "admissionMode"); } } } diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/OwnerBoundStartupMigrationRecoveryPreflight.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/OwnerBoundStartupMigrationRecoveryPreflight.java new file mode 100644 index 0000000000..8e34922920 --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/OwnerBoundStartupMigrationRecoveryPreflight.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.runtime; + +import java.nio.file.Path; +import java.util.Objects; +import org.apache.hertzbeat.manager.maintenance.StandaloneDeploymentOwnerView; +import org.apache.hertzbeat.manager.setup.workflow.ManagedMigrationStartupRecoveryDisposition; +import org.apache.hertzbeat.manager.setup.workflow.ManagedMigrationStartupRecoverySession; + +/** Revalidates the non-owning standalone capability around every recovery attempt. */ +final class OwnerBoundStartupMigrationRecoveryPreflight implements StartupMigrationRecoveryPreflight { + + private final Path canonicalRoot; + private final StandaloneDeploymentOwnerView ownerView; + private final ManagedMigrationStartupRecoverySession session; + + OwnerBoundStartupMigrationRecoveryPreflight( + Path canonicalRoot, + StandaloneDeploymentOwnerView ownerView, + ManagedMigrationStartupRecoverySession session) { + this.canonicalRoot = Objects.requireNonNull(canonicalRoot, "canonicalRoot"); + this.ownerView = Objects.requireNonNull(ownerView, "ownerView"); + this.session = Objects.requireNonNull(session, "session"); + } + + @Override + public ManagedMigrationStartupRecoveryDisposition reconcile() { + requireValidOwner(canonicalRoot, ownerView); + ManagedMigrationStartupRecoveryDisposition disposition = + Objects.requireNonNull(session.reconcile(), "migration recovery disposition"); + requireValidOwner(canonicalRoot, ownerView); + return disposition; + } + + @Override + public void close() { + session.close(); + } + + static void requireValidOwner(Path canonicalRoot, StandaloneDeploymentOwnerView ownerView) { + if (!ownerView.isValid() || !canonicalRoot.equals(ownerView.installationRoot())) { + throw StandaloneDeploymentOwnerException.unavailable(); + } + } +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupCleanup.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupCleanup.java new file mode 100644 index 0000000000..0668768560 --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupCleanup.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.runtime; + +/** Throwable-preserving cleanup arbitration used during pre-Spring startup and shutdown. */ +final class StartupCleanup { + + private StartupCleanup() { + } + + static Result runInterruptSafe(Throwable primary, Runnable cleanup) { + boolean interrupted = Thread.interrupted(); + try { + cleanup.run(); + return new Result(primary, interrupted | Thread.interrupted(), true); + } catch (Throwable failure) { + return new Result(merge(primary, failure), interrupted | Thread.interrupted(), false); + } + } + + static Throwable merge(Throwable primary, Throwable failure) { + if (primary == null) { + return failure; + } + if (failure instanceof Error && !(primary instanceof Error)) { + if (failure != primary) { + failure.addSuppressed(primary); + } + return failure; + } + if (failure != primary) { + primary.addSuppressed(failure); + } + return primary; + } + + static void rethrow(Throwable failure) { + if (failure instanceof Error fatal) { + throw fatal; + } + if (failure instanceof RuntimeException runtime) { + throw runtime; + } + if (failure != null) { + throw new IllegalStateException("Startup cleanup failed"); + } + } + + record Result(Throwable failure, boolean interrupted, boolean completed) { + } +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupMigrationAbortExecutor.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupMigrationAbortExecutor.java new file mode 100644 index 0000000000..61e721c6d9 --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupMigrationAbortExecutor.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.runtime; + +import java.util.concurrent.Executor; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** Process-lifetime bounded daemon lanes for JDBC abort callbacks retained beyond session close. */ +final class StartupMigrationAbortExecutor { + + private static final Executor PROCESS_LIFETIME = new ThreadPoolExecutor( + 2, 2, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>(), + Thread.ofPlatform().daemon().name("hertzbeat-migration-abort-", 0).factory(), + new ThreadPoolExecutor.AbortPolicy()); + + private StartupMigrationAbortExecutor() { + } + + static Executor processLifetime() { + return PROCESS_LIFETIME; + } +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupMigrationPreflightGate.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupMigrationPreflightGate.java new file mode 100644 index 0000000000..d93eeca984 --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupMigrationPreflightGate.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.runtime; + +import java.nio.file.Path; +import java.time.Duration; +import java.util.Objects; +import java.util.concurrent.Executor; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.maintenance.StandaloneDeploymentOwnerView; +import org.apache.hertzbeat.manager.setup.workflow.ManagedMigrationStartupRecoveryDisposition; + +/** Owns the secret-free migration startup decision and its sticky exact-datasource admission. */ +final class StartupMigrationPreflightGate implements AutoCloseable { + + private final Path canonicalRoot; + private final StandaloneDeploymentOwnerView ownerView; + private final StartupMigrationRecoveryPreflight preflight; + private boolean migrationGateActive; + private boolean exactManagedDatasourceRequired; + private boolean reloadRequested; + + private StartupMigrationPreflightGate( + Path canonicalRoot, + StandaloneDeploymentOwnerView ownerView, + StartupMigrationRecoveryPreflight preflight) { + this.canonicalRoot = Objects.requireNonNull(canonicalRoot, "canonicalRoot"); + this.ownerView = Objects.requireNonNull(ownerView, "ownerView"); + this.preflight = Objects.requireNonNull(preflight, "preflight"); + } + + static StartupMigrationPreflightGate open( + Path canonicalRoot, + StandaloneDeploymentOwnerView ownerView, + StartupMigrationRecoveryPreflightFactory factory, + Duration timeout, + Executor abortExecutor) { + requireValidOwner(canonicalRoot, ownerView); + StartupMigrationRecoveryPreflight preflight = Objects.requireNonNull( + factory.open(canonicalRoot, ownerView, timeout, abortExecutor), + "migration recovery preflight"); + return new StartupMigrationPreflightGate(canonicalRoot, ownerView, preflight); + } + + Outcome reconcile() { + requireValidOwner(); + ManagedMigrationStartupRecoveryDisposition disposition = Objects.requireNonNull( + preflight.reconcile(), "migration recovery disposition"); + requireValidOwner(); + return switch (disposition) { + case NO_MIGRATION -> Outcome.PROBE; + case GATED_RECOVERY -> { + migrationGateActive = true; + yield Outcome.GATED_RECOVERY; + } + case RELOAD_FULL_GATED -> { + migrationGateActive = true; + exactManagedDatasourceRequired = true; + yield Outcome.RELOAD_FULL_GATED; + } + }; + } + + void requireValidOwner() { + requireValidOwner(canonicalRoot, ownerView); + } + + void requirePublicTransitionAllowed(StartupDecision decision) { + if (migrationGateActive && decision.mode() == RuntimeMode.NORMAL) { + throw StandaloneDeploymentOwnerException.unavailable(); + } + } + + StartupLaunchAdmission.Mode admissionMode() { + return exactManagedDatasourceRequired + ? StartupLaunchAdmission.Mode.EXACT_MANAGED_DATASOURCE + : StartupLaunchAdmission.Mode.ORDINARY; + } + + boolean exactManagedDatasourceRequired() { + return exactManagedDatasourceRequired; + } + + boolean requiresReconciliationAfterNormal() { + return migrationGateActive; + } + + boolean claimFreshReload() { + boolean fresh = !reloadRequested; + reloadRequested = true; + return fresh; + } + + @Override + public void close() { + preflight.close(); + } + + private static void requireValidOwner(Path root, StandaloneDeploymentOwnerView owner) { + OwnerBoundStartupMigrationRecoveryPreflight.requireValidOwner(root, owner); + } + + enum Outcome { + PROBE, + GATED_RECOVERY, + RELOAD_FULL_GATED + } +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupMigrationRecoveryPreflight.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupMigrationRecoveryPreflight.java new file mode 100644 index 0000000000..328a40905e --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupMigrationRecoveryPreflight.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.runtime; + +import org.apache.hertzbeat.manager.setup.workflow.ManagedMigrationStartupRecoveryDisposition; + +/** One owner-bound, long-lived migration recovery session used before Spring starts. */ +interface StartupMigrationRecoveryPreflight extends AutoCloseable { + + ManagedMigrationStartupRecoveryDisposition reconcile(); + + @Override + void close(); +} diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupMigrationRecoveryPreflightFactory.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupMigrationRecoveryPreflightFactory.java new file mode 100644 index 0000000000..7421cc2020 --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/StartupMigrationRecoveryPreflightFactory.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.runtime; + +import java.nio.file.Path; +import java.time.Duration; +import java.util.concurrent.Executor; +import org.apache.hertzbeat.manager.maintenance.StandaloneDeploymentOwnerView; +import org.apache.hertzbeat.manager.setup.workflow.ManagedMigrationStartupRecoverySession; + +/** Creates one migration recovery session after the standalone owner is established. */ +@FunctionalInterface +interface StartupMigrationRecoveryPreflightFactory { + + StartupMigrationRecoveryPreflight open( + Path canonicalRoot, + StandaloneDeploymentOwnerView ownerView, + Duration verificationTimeout, + Executor abortExecutor); + + static StartupMigrationRecoveryPreflightFactory system() { + return (root, owner, timeout, executor) -> { + OwnerBoundStartupMigrationRecoveryPreflight.requireValidOwner(root, owner); + ManagedMigrationStartupRecoverySession session = + new ManagedMigrationStartupRecoverySession(root, timeout, executor); + return new OwnerBoundStartupMigrationRecoveryPreflight(root, owner, session); + }; + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupMigrationLifecycleTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupMigrationLifecycleTest.java new file mode 100644 index 0000000000..c795a2ff07 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupMigrationLifecycleTest.java @@ -0,0 +1,339 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.runtime; + +import static org.apache.hertzbeat.startup.runtime.StartupMigrationPreflightTestSupport.coordinator; +import static org.apache.hertzbeat.startup.runtime.StartupMigrationPreflightTestSupport.replaceOwnerLock; +import static org.apache.hertzbeat.startup.runtime.StartupMigrationPreflightTestSupport.rootArgs; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.maintenance.StandaloneDeploymentOwnerView; +import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; +import org.apache.hertzbeat.manager.setup.workflow.ManagedMigrationStartupRecoveryDisposition; +import org.apache.hertzbeat.startup.runtime.StartupMigrationPreflightTestSupport.AdmittedLauncherAdapter; +import org.apache.hertzbeat.startup.runtime.StartupMigrationPreflightTestSupport.RecordingAdmittedLauncher; +import org.apache.hertzbeat.startup.runtime.StartupMigrationPreflightTestSupport.RecordingContext; +import org.apache.hertzbeat.startup.runtime.StartupMigrationPreflightTestSupport.RecordingPreflight; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HertzBeatStartupMigrationLifecycleTest { + + @TempDir + private Path installationRoot; + + @Test + void queuedCallbackDoesNotReviveClosedCoordinatorAfterOwnerRelease() { + List events = new ArrayList<>(); + AtomicInteger probes = new AtomicInteger(); + RecordingPreflight preflight = new RecordingPreflight(events, + ManagedMigrationStartupRecoveryDisposition.GATED_RECOVERY, + ManagedMigrationStartupRecoveryDisposition.RELOAD_FULL_GATED, + ManagedMigrationStartupRecoveryDisposition.NO_MIGRATION); + RecordingAdmittedLauncher launcher = new RecordingAdmittedLauncher(events); + HertzBeatStartupCoordinator coordinator = coordinator( + installationRoot, + ignored -> { + probes.incrementAndGet(); + return StartupDecision.normal(); + }, launcher, events, (root, owner, timeout, executor) -> preflight); + coordinator.start(rootArgs(installationRoot)); + coordinator.configurationApplied(); + SetupRuntimeTransition queued = launcher.transitions.getLast(); + int launches = launcher.modes.size(); + coordinator.close(); + + queued.configurationApplied(); + + assertEquals(0, probes.get()); + assertEquals(launches, launcher.modes.size()); + assertNull(coordinator.currentContext()); + assertThrows(StandaloneDeploymentOwnerException.class, + () -> coordinator.transition(StartupDecision.normal())); + assertEquals(0, probes.get()); + assertEquals(launches, launcher.modes.size()); + } + + @Test + void queuedCompletionDoesNotReviveNormalWhileClosedCleanupIsPending() { + List events = new ArrayList<>(); + AtomicInteger contextCloses = new AtomicInteger(); + RecordingPreflight preflight = new RecordingPreflight(events, + ManagedMigrationStartupRecoveryDisposition.NO_MIGRATION); + RecordingAdmittedLauncher launcher = new RecordingAdmittedLauncher(events) { + @Override + RunningApplicationContext context( + StartupDecision decision, StandaloneDeploymentOwnerView view) { + if (decision.mode() != RuntimeMode.FULL_SETUP_GATED) { + return super.context(decision, view); + } + return new RecordingContext(decision.mode(), events) { + @Override + public void close() { + events.add("close-attempt:" + mode().value()); + if (contextCloses.getAndIncrement() == 0) { + throw new IllegalStateException("close pending"); + } + super.close(); + } + }; + } + }; + HertzBeatStartupCoordinator coordinator = coordinator( + installationRoot, + ignored -> new StartupDecision(RuntimeMode.FULL_SETUP_GATED), + launcher, events, (root, owner, timeout, executor) -> preflight); + RunningApplicationContext full = coordinator.start(rootArgs(installationRoot)); + SetupRuntimeTransition queued = launcher.transitions.getLast(); + int launches = launcher.modes.size(); + + assertThrows(IllegalStateException.class, coordinator::close); + queued.completeSetup(); + + assertSame(full, coordinator.currentContext()); + assertTrue(full.isActive()); + assertEquals(launches, launcher.modes.size()); + coordinator.close(); + assertNull(coordinator.currentContext()); + assertEquals(2, contextCloses.get()); + } + + @Test + void trustedCompletionRevalidatesOwnerBeforeClosingTheFullContext() { + List events = new ArrayList<>(); + RecordingPreflight preflight = new RecordingPreflight(events, + ManagedMigrationStartupRecoveryDisposition.GATED_RECOVERY, + ManagedMigrationStartupRecoveryDisposition.RELOAD_FULL_GATED); + RecordingAdmittedLauncher launcher = new RecordingAdmittedLauncher(events); + HertzBeatStartupCoordinator coordinator = coordinator( + installationRoot, + ignored -> { + throw new AssertionError("migration recovery must not probe"); + }, launcher, events, (root, owner, timeout, executor) -> preflight); + coordinator.start(rootArgs(installationRoot)); + coordinator.configurationApplied(); + RunningApplicationContext full = coordinator.currentContext(); + SetupRuntimeTransition completion = launcher.transitions.getLast(); + replaceOwnerLock(installationRoot); + + assertThrows(StandaloneDeploymentOwnerException.class, completion::completeSetup); + + assertSame(full, coordinator.currentContext()); + assertTrue(full.isActive()); + assertEquals(List.of(RuntimeMode.RECOVERY, RuntimeMode.FULL_SETUP_GATED), launcher.modes); + coordinator.close(); + } + + @Test + void transitionRetriesTheExactOldContextCloseBeforeLaunchingAnythingNew() { + List events = new ArrayList<>(); + AtomicInteger probes = new AtomicInteger(); + AtomicInteger closes = new AtomicInteger(); + RecordingPreflight preflight = new RecordingPreflight(events, + ManagedMigrationStartupRecoveryDisposition.NO_MIGRATION); + RecordingAdmittedLauncher launcher = new RecordingAdmittedLauncher(events) { + @Override + RunningApplicationContext context( + StartupDecision decision, StandaloneDeploymentOwnerView view) { + if (decision.mode() != RuntimeMode.FULL_SETUP_GATED) { + return super.context(decision, view); + } + return new RecordingContext(decision.mode(), events) { + @Override + public void close() { + events.add("close-attempt:" + mode().value()); + if (closes.getAndIncrement() == 0) { + throw new IllegalStateException("close pending"); + } + super.close(); + } + }; + } + }; + HertzBeatStartupCoordinator coordinator = coordinator( + installationRoot, + ignored -> probes.getAndIncrement() == 0 + ? new StartupDecision(RuntimeMode.FULL_SETUP_GATED) + : StartupDecision.normal(), + launcher, events, (root, owner, timeout, executor) -> preflight); + + RunningApplicationContext original = coordinator.start(rootArgs(installationRoot)); + assertThrows(IllegalStateException.class, coordinator::configurationApplied); + assertSame(original, coordinator.currentContext()); + assertEquals(1, launcher.modes.size()); + coordinator.configurationApplied(); + + assertEquals(RuntimeMode.NORMAL, coordinator.mode()); + assertEquals(List.of(RuntimeMode.FULL_SETUP_GATED, RuntimeMode.NORMAL), launcher.modes); + coordinator.close(); + } + + @Test + void closeRetriesOnlyResourcesWhoseExactCloseDidNotComplete() { + List events = new ArrayList<>(); + AtomicInteger contextCloses = new AtomicInteger(); + AtomicInteger preflightCloses = new AtomicInteger(); + RecordingPreflight preflight = new RecordingPreflight(events, + ManagedMigrationStartupRecoveryDisposition.NO_MIGRATION) { + @Override + public void close() { + events.add("preflight-close"); + if (preflightCloses.getAndIncrement() < 2) { + throw new IllegalStateException("preflight close pending"); + } + } + }; + RecordingAdmittedLauncher launcher = new RecordingAdmittedLauncher(events) { + @Override + RunningApplicationContext context( + StartupDecision decision, StandaloneDeploymentOwnerView view) { + return new RecordingContext(decision.mode(), events) { + @Override + public void close() { + events.add("close:" + mode().value()); + if (contextCloses.getAndIncrement() == 0) { + throw new IllegalArgumentException("context close pending"); + } + } + }; + } + }; + HertzBeatStartupCoordinator coordinator = coordinator( + installationRoot, ignored -> StartupDecision.normal(), launcher, events, + (root, owner, timeout, executor) -> preflight); + coordinator.start(rootArgs(installationRoot)); + + IllegalArgumentException first = assertThrows(IllegalArgumentException.class, coordinator::close); + assertEquals(1, first.getSuppressed().length); + assertEquals("preflight close pending", first.getSuppressed()[0].getMessage()); + ResolvedStartupInstallationRoot root = new StartupInstallationRootResolver() + .resolve(rootArgs(installationRoot)); + assertThrows(StandaloneDeploymentOwnerException.class, + () -> StandaloneDeploymentOwner.acquire(root)); + assertThrows(IllegalStateException.class, coordinator::close); + assertThrows(StandaloneDeploymentOwnerException.class, + () -> StandaloneDeploymentOwner.acquire(root)); + coordinator.close(); + + assertEquals(2, contextCloses.get()); + assertEquals(3, preflightCloses.get()); + try (StandaloneDeploymentOwner ignored = StandaloneDeploymentOwner.acquire(root)) { + assertTrue(ignored.isValid()); + } + assertThrows(StandaloneDeploymentOwnerException.class, + () -> coordinator.start(rootArgs(installationRoot))); + } + + @Test + void closePromotesPreflightErrorOverContextRuntimeAndIsolatesInterruptsBetweenActions() { + List events = new ArrayList<>(); + IllegalStateException contextFailure = new IllegalStateException("context-close"); + AssertionError preflightFailure = new AssertionError("preflight-close"); + AtomicInteger contextCloses = new AtomicInteger(); + AtomicInteger preflightCloses = new AtomicInteger(); + RecordingPreflight preflight = new RecordingPreflight(events, + ManagedMigrationStartupRecoveryDisposition.NO_MIGRATION) { + @Override + public void close() { + events.add("preflight-close"); + assertFalse(Thread.currentThread().isInterrupted()); + Thread.interrupted(); + if (preflightCloses.getAndIncrement() == 0) { + throw preflightFailure; + } + } + }; + RecordingAdmittedLauncher launcher = new RecordingAdmittedLauncher(events) { + @Override + RunningApplicationContext context( + StartupDecision decision, StandaloneDeploymentOwnerView view) { + return new RecordingContext(decision.mode(), events) { + @Override + public void close() { + events.add("close:" + mode().value()); + Thread.currentThread().interrupt(); + if (contextCloses.getAndIncrement() == 0) { + throw contextFailure; + } + } + }; + } + }; + HertzBeatStartupCoordinator coordinator = coordinator( + installationRoot, ignored -> StartupDecision.normal(), launcher, events, + (root, owner, timeout, executor) -> preflight); + coordinator.start(rootArgs(installationRoot)); + + Thread.currentThread().interrupt(); + try { + AssertionError thrown = assertThrows(AssertionError.class, coordinator::close); + assertSame(preflightFailure, thrown); + assertEquals(1, thrown.getSuppressed().length); + assertSame(contextFailure, thrown.getSuppressed()[0]); + assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); + } + try { + coordinator.close(); + assertEquals(2, contextCloses.get()); + assertEquals(2, preflightCloses.get()); + assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); + } + ResolvedStartupInstallationRoot root = new StartupInstallationRootResolver() + .resolve(rootArgs(installationRoot)); + try (StandaloneDeploymentOwner owner = StandaloneDeploymentOwner.acquire(root)) { + assertTrue(owner.isValid()); + } + } + + @Test + void recoveryLaunchErrorOutranksAndRetainsTheOriginalRuntimeFailure() { + List events = new ArrayList<>(); + AssertionError fatal = new AssertionError("recovery fatal"); + StartupContextLauncher launcher = new StartupContextLauncher() { + @Override + public RunningApplicationContext launch( + StartupDecision decision, String[] args, SetupRuntimeTransition transition) { + throw new AssertionError("owned startup must use admitted launch"); + } + }; + AdmittedStartupContextLauncher admitted = ( + decision, args, transition, root, view, mode) -> { + if (decision.mode() == RuntimeMode.RECOVERY) { + throw fatal; + } + throw new IllegalStateException("normal launch failed"); + }; + StartupContextLauncher both = new AdmittedLauncherAdapter(launcher, admitted); + RecordingPreflight preflight = new RecordingPreflight(events, + ManagedMigrationStartupRecoveryDisposition.NO_MIGRATION); + HertzBeatStartupCoordinator coordinator = coordinator( + installationRoot, ignored -> StartupDecision.normal(), both, events, + (root, owner, timeout, executor) -> preflight); + + AssertionError thrown = assertThrows(AssertionError.class, + () -> coordinator.start(rootArgs(installationRoot))); + + assertSame(fatal, thrown); + assertEquals(1, thrown.getSuppressed().length); + assertEquals("normal launch failed", thrown.getSuppressed()[0].getMessage()); + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupMigrationPreflightFlowTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupMigrationPreflightFlowTest.java new file mode 100644 index 0000000000..b37620e89a --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupMigrationPreflightFlowTest.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.runtime; + +import static org.apache.hertzbeat.startup.runtime.StartupMigrationPreflightTestSupport.CALLER_OWNED_ABORT_EXECUTOR; +import static org.apache.hertzbeat.startup.runtime.StartupMigrationPreflightTestSupport.PREFLIGHT_TIMEOUT; +import static org.apache.hertzbeat.startup.runtime.StartupMigrationPreflightTestSupport.coordinator; +import static org.apache.hertzbeat.startup.runtime.StartupMigrationPreflightTestSupport.replaceOwnerLock; +import static org.apache.hertzbeat.startup.runtime.StartupMigrationPreflightTestSupport.rootArgs; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.setup.config.SetupInstallationPaths; +import org.apache.hertzbeat.manager.setup.workflow.ManagedMigrationStartupRecoveryDisposition; +import org.apache.hertzbeat.startup.runtime.StartupMigrationPreflightTestSupport.RecordingAdmittedLauncher; +import org.apache.hertzbeat.startup.runtime.StartupMigrationPreflightTestSupport.RecordingPreflight; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HertzBeatStartupMigrationPreflightFlowTest { + + @TempDir + private Path installationRoot; + + @Test + void opensOneOwnerBoundPreflightBeforeProbeAndPinsTerminalNormal() throws Exception { + List events = new ArrayList<>(); + RecordingPreflight preflight = new RecordingPreflight(events, + ManagedMigrationStartupRecoveryDisposition.NO_MIGRATION, + ManagedMigrationStartupRecoveryDisposition.NO_MIGRATION); + Path canonicalRoot = installationRoot.toRealPath(); + AtomicInteger opens = new AtomicInteger(); + StartupMigrationRecoveryPreflightFactory preflightFactory = (root, owner, timeout, executor) -> { + events.add("preflight-open"); + opens.incrementAndGet(); + assertEquals(canonicalRoot, root); + assertEquals(root, owner.installationRoot()); + assertTrue(owner.isValid()); + assertEquals(PREFLIGHT_TIMEOUT, timeout); + assertSame(CALLER_OWNED_ABORT_EXECUTOR, executor); + return preflight; + }; + RecordingAdmittedLauncher launcher = new RecordingAdmittedLauncher(events); + HertzBeatStartupCoordinator coordinator = coordinator( + installationRoot, + ignored -> { + events.add("probe"); + return StartupDecision.normal(); + }, launcher, events, preflightFactory); + + coordinator.start(rootArgs(installationRoot)); + coordinator.configurationApplied(); + + assertEquals(1, opens.get()); + assertEquals(List.of( + "owner", "preflight-open", "reconcile", "probe", "open:normal:ORDINARY"), events); + coordinator.close(); + assertEquals(List.of("close:normal", "preflight-close"), events.subList(events.size() - 2, events.size())); + } + + @Test + void unresolvedMigrationOverridesForcedNormalAndReloadsFreshExactManagedContexts() { + List events = new ArrayList<>(); + RecordingPreflight preflight = new RecordingPreflight(events, + ManagedMigrationStartupRecoveryDisposition.GATED_RECOVERY, + ManagedMigrationStartupRecoveryDisposition.RELOAD_FULL_GATED); + AtomicInteger probes = new AtomicInteger(); + RecordingAdmittedLauncher launcher = new RecordingAdmittedLauncher(events); + HertzBeatStartupCoordinator coordinator = coordinator( + installationRoot, + ignored -> { + probes.incrementAndGet(); + return StartupDecision.normal(); + }, launcher, events, (root, owner, timeout, executor) -> preflight); + + coordinator.start(new String[] { + "--" + SetupInstallationPaths.ROOT_PROPERTY + "=" + installationRoot, + "--" + StartupModePropertyProbe.PROPERTY_NAME + "=" + RuntimeMode.NORMAL.value() + }); + assertEquals(RuntimeMode.RECOVERY, coordinator.mode()); + coordinator.configurationApplied(); + assertEquals(RuntimeMode.FULL_SETUP_GATED, coordinator.mode()); + assertThrows(StandaloneDeploymentOwnerException.class, + () -> coordinator.transition(StartupDecision.normal())); + launcher.transitions.getLast().completeSetup(); + + assertEquals(0, probes.get()); + assertEquals(List.of( + "owner", "reconcile", "open:recovery:ORDINARY", + "reconcile", "close:recovery", "open:full_setup_gated:EXACT_MANAGED_DATASOURCE", + "close:full_setup_gated", "open:normal:EXACT_MANAGED_DATASOURCE"), events); + assertEquals(RuntimeMode.NORMAL, coordinator.mode()); + assertTrue(launcher.views.getLast().isValid()); + coordinator.close(); + } + + @Test + void ownerReplacementAfterReconcilePreventsProbeAndSpringLaunch() throws Exception { + List events = new ArrayList<>(); + AtomicInteger probes = new AtomicInteger(); + RecordingAdmittedLauncher launcher = new RecordingAdmittedLauncher(events); + StartupMigrationRecoveryPreflight replacing = new StartupMigrationRecoveryPreflight() { + @Override + public ManagedMigrationStartupRecoveryDisposition reconcile() { + events.add("reconcile"); + replaceOwnerLock(installationRoot); + return ManagedMigrationStartupRecoveryDisposition.NO_MIGRATION; + } + + @Override + public void close() { + events.add("preflight-close"); + } + }; + HertzBeatStartupCoordinator coordinator = coordinator( + installationRoot, + ignored -> { + probes.incrementAndGet(); + return StartupDecision.normal(); + }, launcher, events, (root, owner, timeout, executor) -> replacing); + + assertThrows(StandaloneDeploymentOwnerException.class, + () -> coordinator.start(rootArgs(installationRoot))); + + assertEquals(0, probes.get()); + assertTrue(launcher.modes.isEmpty()); + assertEquals(List.of("owner", "reconcile", "preflight-close"), events); + ResolvedStartupInstallationRoot root = new StartupInstallationRootResolver() + .resolve(rootArgs(installationRoot)); + try (StandaloneDeploymentOwner owner = StandaloneDeploymentOwner.acquire(root)) { + assertTrue(owner.isValid()); + } + } + + @Test + void failedPostOpenOwnerCheckRetainsTheExactPreflightUntilCloseRetry() { + List events = new ArrayList<>(); + AtomicInteger closes = new AtomicInteger(); + RecordingPreflight preflight = new RecordingPreflight(events, + ManagedMigrationStartupRecoveryDisposition.NO_MIGRATION) { + @Override + public void close() { + events.add("preflight-close"); + if (closes.getAndIncrement() == 0) { + throw new IllegalStateException("close pending"); + } + } + }; + RecordingAdmittedLauncher launcher = new RecordingAdmittedLauncher(events); + HertzBeatStartupCoordinator coordinator = coordinator( + installationRoot, + ignored -> { + throw new AssertionError("probe must not run"); + }, launcher, events, (root, owner, timeout, executor) -> { + replaceOwnerLock(installationRoot); + return preflight; + }); + + assertThrows(StandaloneDeploymentOwnerException.class, + () -> coordinator.start(rootArgs(installationRoot))); + assertEquals(1, closes.get()); + coordinator.close(); + + assertEquals(2, closes.get()); + try (StandaloneDeploymentOwner owner = StandaloneDeploymentOwner.acquire( + new StartupInstallationRootResolver().resolve(rootArgs(installationRoot)))) { + assertTrue(owner.isValid()); + } + } + + @Test + void repeatedReloadDispositionKeepsTheFreshExactFullContext() { + List events = new ArrayList<>(); + RecordingPreflight preflight = new RecordingPreflight(events, + ManagedMigrationStartupRecoveryDisposition.GATED_RECOVERY, + ManagedMigrationStartupRecoveryDisposition.RELOAD_FULL_GATED, + ManagedMigrationStartupRecoveryDisposition.RELOAD_FULL_GATED); + RecordingAdmittedLauncher launcher = new RecordingAdmittedLauncher(events); + HertzBeatStartupCoordinator coordinator = coordinator( + installationRoot, + ignored -> { + throw new AssertionError("migration recovery must not probe"); + }, launcher, events, (root, owner, timeout, executor) -> preflight); + + coordinator.start(rootArgs(installationRoot)); + coordinator.configurationApplied(); + RunningApplicationContext full = coordinator.currentContext(); + coordinator.configurationApplied(); + + assertSame(full, coordinator.currentContext()); + assertEquals(1, launcher.modes.stream() + .filter(RuntimeMode.FULL_SETUP_GATED::equals) + .count()); + coordinator.close(); + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupMigrationPreflightTestSupport.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupMigrationPreflightTestSupport.java new file mode 100644 index 0000000000..ed2f559bed --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/StartupMigrationPreflightTestSupport.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.startup.runtime; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.List; +import java.util.concurrent.Executor; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.maintenance.StandaloneDeploymentOwnerView; +import org.apache.hertzbeat.manager.setup.config.SetupInstallationPaths; +import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; +import org.apache.hertzbeat.manager.setup.workflow.ManagedMigrationStartupRecoveryDisposition; + +final class StartupMigrationPreflightTestSupport { + + static final Duration PREFLIGHT_TIMEOUT = Duration.ofSeconds(7); + static final Executor CALLER_OWNED_ABORT_EXECUTOR = Runnable::run; + + private StartupMigrationPreflightTestSupport() { + } + + static HertzBeatStartupCoordinator coordinator( + Path installationRoot, + StartupDecisionProbe probe, + StartupContextLauncher launcher, + List events, + StartupMigrationRecoveryPreflightFactory preflightFactory) { + StartupInstallationRootResolver resolver = new StartupInstallationRootResolver(); + StandaloneDeploymentOwnerFactory ownerFactory = root -> { + events.add("owner"); + return StandaloneDeploymentOwner.acquire(root); + }; + return new HertzBeatStartupCoordinator( + probe, launcher, new StartupFailureReporter(), resolver, ownerFactory, + preflightFactory, PREFLIGHT_TIMEOUT, CALLER_OWNED_ABORT_EXECUTOR); + } + + static String[] rootArgs(Path installationRoot) { + return new String[] {"--" + SetupInstallationPaths.ROOT_PROPERTY + "=" + installationRoot}; + } + + static void replaceOwnerLock(Path installationRoot) { + Path lock = installationRoot.resolve(StandaloneDeploymentOwner.LOCK_PATH); + try { + Files.delete(lock); + Files.writeString(lock, "replacement\n", StandardCharsets.UTF_8); + Files.setPosixFilePermissions(lock, PosixFilePermissions.fromString("rw-------")); + } catch (IOException failure) { + throw new AssertionError("test owner replacement failed", failure); + } + } + + static class RecordingPreflight implements StartupMigrationRecoveryPreflight { + + private final List events; + private final Deque dispositions; + + RecordingPreflight( + List events, ManagedMigrationStartupRecoveryDisposition... dispositions) { + this.events = events; + this.dispositions = new ArrayDeque<>(List.of(dispositions)); + } + + @Override + public ManagedMigrationStartupRecoveryDisposition reconcile() { + events.add("reconcile"); + return dispositions.size() == 1 ? dispositions.getFirst() : dispositions.removeFirst(); + } + + @Override + public void close() { + events.add("preflight-close"); + } + } + + static class RecordingAdmittedLauncher + implements StartupContextLauncher, AdmittedStartupContextLauncher { + + private final List events; + final List modes = new ArrayList<>(); + final List transitions = new ArrayList<>(); + final List views = new ArrayList<>(); + + RecordingAdmittedLauncher(List events) { + this.events = events; + } + + @Override + public RunningApplicationContext launch( + StartupDecision decision, String[] args, SetupRuntimeTransition transition) { + throw new AssertionError("owned startup must use admitted launch"); + } + + @Override + public RunningApplicationContext launchAdmitted( + StartupDecision decision, + String[] args, + SetupRuntimeTransition transition, + Path root, + StandaloneDeploymentOwnerView view, + StartupLaunchAdmission.Mode admissionMode) { + events.add("open:" + decision.mode().value() + ":" + admissionMode); + modes.add(decision.mode()); + transitions.add(transition); + views.add(view); + return context(decision, view); + } + + RunningApplicationContext context( + StartupDecision decision, StandaloneDeploymentOwnerView view) { + return new RecordingContext(decision.mode(), events); + } + } + + static class RecordingContext implements RunningApplicationContext { + + private final RuntimeMode mode; + private final List events; + private boolean active = true; + + RecordingContext(RuntimeMode mode, List events) { + this.mode = mode; + this.events = events; + } + + @Override + public RuntimeMode mode() { + return mode; + } + + @Override + public boolean isActive() { + return active; + } + + @Override + public void close() { + active = false; + events.add("close:" + mode.value()); + } + } + + static final class AdmittedLauncherAdapter + implements StartupContextLauncher, AdmittedStartupContextLauncher { + + private final StartupContextLauncher publicLauncher; + private final AdmittedStartupContextLauncher admittedLauncher; + + AdmittedLauncherAdapter( + StartupContextLauncher publicLauncher, AdmittedStartupContextLauncher admittedLauncher) { + this.publicLauncher = publicLauncher; + this.admittedLauncher = admittedLauncher; + } + + @Override + public RunningApplicationContext launch( + StartupDecision decision, String[] args, SetupRuntimeTransition transition) { + return publicLauncher.launch(decision, args, transition); + } + + @Override + public RunningApplicationContext launchAdmitted( + StartupDecision decision, + String[] args, + SetupRuntimeTransition transition, + Path root, + StandaloneDeploymentOwnerView view, + StartupLaunchAdmission.Mode mode) { + return admittedLauncher.launchAdmitted(decision, args, transition, root, view, mode); + } + } +} From ae4c15d490e024c91c0966ffa0b8dc6cf9d2b4d5 Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 19:14:23 +0800 Subject: [PATCH 64/71] Bind migration requests and prepared exports --- .../setup/api/DeploymentApiContract.java | 8 +- .../setup/api/DeploymentController.java | 62 +++-- .../manager/setup/api/DeploymentWorkflow.java | 4 +- .../BoundedMigrationExportBuffer.java | 99 +++++++ .../workflow/MigrationExportRenderer.java | 20 +- .../NonClosingMigrationExportStream.java | 42 +++ .../workflow/PreparedMigrationExport.java | 121 +++++++++ .../PreparedMigrationExportException.java | 16 ++ .../setup/workflow/StagedMigrationExport.java | 85 ++++++ .../setup/api/DeploymentApiContractTest.java | 27 +- .../api/DeploymentControllerExportTest.java | 248 ++++++++++++++++++ .../DeploymentControllerRegistrationTest.java | 4 +- .../setup/api/DeploymentControllerTest.java | 128 +++------ .../setup/api/SetupApiContractTest.java | 5 +- .../BoundedMigrationExportBufferTest.java | 51 ++++ .../workflow/PreparedMigrationExportTest.java | 208 +++++++++++++++ 16 files changed, 981 insertions(+), 147 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/BoundedMigrationExportBuffer.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/NonClosingMigrationExportStream.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/PreparedMigrationExport.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/PreparedMigrationExportException.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/StagedMigrationExport.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerExportTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/BoundedMigrationExportBufferTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/PreparedMigrationExportTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContract.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContract.java index 7729fa81b7..c74b1e7633 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContract.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContract.java @@ -196,18 +196,22 @@ public final class DeploymentApiContract { /** H2-to-external-database migration input. */ public record MetadataMigrationRequest( + @NotBlank String operationId, @NotNull MigrationTarget target, @NotNull @Valid MetadataDatabaseConfiguration targetDatabase, @NotNull ApplyMode applyMode) { public MetadataMigrationRequest { + if (!OperationIdValidator.isSafe(operationId)) { + throw new IllegalArgumentException("Invalid migration operation id"); + } MigrationContractValidator.validateTarget(target, targetDatabase); } @Override public String toString() { - return "MetadataMigrationRequest[target=" + target + ", targetDatabase=, applyMode=" - + applyMode + "]"; + return "MetadataMigrationRequest[operationId=" + operationId + ", target=" + target + + ", targetDatabase=, applyMode=" + applyMode + "]"; } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentController.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentController.java index 40f51dbc22..b46dcd7c1f 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentController.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentController.java @@ -17,7 +17,9 @@ package org.apache.hertzbeat.manager.setup.api; +import jakarta.servlet.http.HttpServletResponse; import jakarta.validation.Valid; +import java.io.IOException; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.ActivateMigrationRequest; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentView; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MetadataMigrationRequest; @@ -26,9 +28,9 @@ import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationExp import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationView; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationResponse; -import org.apache.hertzbeat.manager.setup.workflow.MigrationExportRenderer; +import org.apache.hertzbeat.manager.setup.workflow.PreparedMigrationExport; +import org.apache.hertzbeat.manager.setup.workflow.StagedMigrationExport; import org.springframework.beans.factory.ObjectProvider; -import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; @@ -36,20 +38,15 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; /** Transport-only adapter that fails safely until the migration workflow is available. */ @RestController public final class DeploymentController { private final ObjectProvider workflowProvider; - private final ObjectProvider rendererProvider; - public DeploymentController( - ObjectProvider workflowProvider, - ObjectProvider rendererProvider) { + public DeploymentController(ObjectProvider workflowProvider) { this.workflowProvider = workflowProvider; - this.rendererProvider = rendererProvider; } @GetMapping(DeploymentApiContract.DEPLOYMENT_PATH) @@ -86,16 +83,25 @@ public final class DeploymentController { } @PostMapping(DeploymentApiContract.EXPORT_PATH) - public ResponseEntity export( - @PathVariable String operationId, @Valid @RequestBody MigrationExportRequest request) { + public void export( + @PathVariable String operationId, @Valid @RequestBody MigrationExportRequest request, + HttpServletResponse response) throws IOException { requireOperationId(operationId); - MigrationExportRenderer renderer = renderer(); - ExportResponse metadata = workflow().prepareExport(operationId, request); - StreamingResponseBody body = output -> renderer.write(operationId, request, output); - return SetupHttpContract.noStore() - .header(HttpHeaders.CONTENT_DISPOSITION, - "attachment; filename=\"" + metadata.fileName() + "\"") - .header(HttpHeaders.CONTENT_TYPE, metadata.mediaType()).body(body); + boolean responseMutated = false; + try (PreparedMigrationExport prepared = workflow().prepareExport(operationId, request); + StagedMigrationExport staged = prepared.stage()) { + ExportResponse metadata = staged.metadata(); + responseMutated = true; + response.setHeader("Cache-Control", "no-store"); + response.setHeader("Content-Disposition", "attachment; filename=\"" + metadata.fileName() + "\""); + response.setContentType(metadata.mediaType()); + response.setContentLength(staged.size()); + staged.writeTo(response.getOutputStream()); + response.flushBuffer(); + } catch (IOException | RuntimeException failure) { + resetUncommitted(response, responseMutated); + throw failure; + } } private DeploymentWorkflow workflow() { @@ -112,16 +118,22 @@ public final class DeploymentController { } } - private MigrationExportRenderer renderer() { - MigrationExportRenderer renderer = rendererProvider.getIfUnique(); - if (renderer == null) { - throw unavailable(); - } - return renderer; - } - private SetupApiException unavailable() { return new SetupApiException( SetupApiContract.SetupErrorCode.MIGRATION_UNAVAILABLE, HttpStatus.SERVICE_UNAVAILABLE); } + + private void resetUncommitted(HttpServletResponse response, boolean responseMutated) { + if (!responseMutated) { + return; + } + try { + if (!response.isCommitted()) { + response.reset(); + } + } catch (RuntimeException ignored) { + // Preserve the original safe transport failure; response rollback is best effort. + } + } + } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentWorkflow.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentWorkflow.java index 4bf48b1ece..90d218e14a 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentWorkflow.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/DeploymentWorkflow.java @@ -23,8 +23,8 @@ import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MetadataMigr import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MetadataMigrationValidationRequest; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationExportRequest; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationView; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationResponse; +import org.apache.hertzbeat.manager.setup.workflow.PreparedMigrationExport; /** Authenticated deployment boundary implemented by a later migration engine milestone. */ public interface DeploymentWorkflow { @@ -39,5 +39,5 @@ public interface DeploymentWorkflow { MigrationView activate(String operationId, ActivateMigrationRequest request); - ExportResponse prepareExport(String operationId, MigrationExportRequest request); + PreparedMigrationExport prepareExport(String operationId, MigrationExportRequest request); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/BoundedMigrationExportBuffer.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/BoundedMigrationExportBuffer.java new file mode 100644 index 0000000000..30c688c422 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/BoundedMigrationExportBuffer.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportResponse; + +/** Bounded in-memory staging buffer whose backing bytes have explicit ownership. */ +final class BoundedMigrationExportBuffer extends OutputStream implements AutoCloseable { + + private static final int INITIAL_CAPACITY = 1024; + + private final int limit; + private final PayloadFactory payloadFactory; + private byte[] bytes; + private int size; + private boolean transferred; + private boolean closed; + + BoundedMigrationExportBuffer(int limit) { + this(limit, StagedMigrationExport::new); + } + + BoundedMigrationExportBuffer(int limit, PayloadFactory payloadFactory) { + if (limit < 1) { + throw new IllegalArgumentException("limit"); + } + this.limit = limit; + this.payloadFactory = Objects.requireNonNull(payloadFactory, "payloadFactory"); + bytes = new byte[Math.min(limit, INITIAL_CAPACITY)]; + } + + @Override + public void write(int value) throws IOException { + requireCapacity(1); + bytes[size++] = (byte) value; + } + + @Override + public void write(byte[] source, int offset, int length) throws IOException { + Objects.checkFromIndexSize(offset, length, source.length); + requireCapacity(length); + System.arraycopy(source, offset, bytes, size, length); + size += length; + } + + StagedMigrationExport finish(ExportResponse metadata) { + if (transferred || closed) { + throw new PreparedMigrationExportException(); + } + try { + StagedMigrationExport staged = payloadFactory.create(metadata, bytes, size); + transferred = true; + bytes = new byte[0]; + return staged; + } catch (RuntimeException | Error failure) { + close(); + throw failure; + } + } + + @Override + public void close() { + if (closed) { + return; + } + Arrays.fill(bytes, (byte) 0); + bytes = new byte[0]; + size = 0; + closed = true; + } + + private void requireCapacity(int additional) throws IOException { + if (transferred || closed || additional > limit - size) { + throw new IOException("Prepared migration export exceeded its size limit"); + } + int required = size + additional; + if (required <= bytes.length) { + return; + } + int nextCapacity = Math.min(limit, Math.max(required, bytes.length * 2)); + byte[] next = Arrays.copyOf(bytes, nextCapacity); + Arrays.fill(bytes, (byte) 0); + bytes = next; + } + + @FunctionalInterface + interface PayloadFactory { + StagedMigrationExport create(ExportResponse metadata, byte[] bytes, int size); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationExportRenderer.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationExportRenderer.java index fe1f6bd143..85c07891d1 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationExportRenderer.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationExportRenderer.java @@ -1,29 +1,19 @@ /* * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with + * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * The ASF licenses this file to You under the Apache License, Version 2.0. */ package org.apache.hertzbeat.manager.setup.workflow; import java.io.IOException; import java.io.OutputStream; -import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationExportRequest; +import org.apache.hertzbeat.manager.setup.config.SecretValue; -/** Streaming port for one-shot external-apply content; implementations must not retain request secrets. */ +/** Pure bound renderer with no owned resources; the borrowed secret must not be retained. */ @FunctionalInterface public interface MigrationExportRenderer { - void write(String operationId, MigrationExportRequest request, OutputStream output) throws IOException; + void write(SecretValue borrowedSecret, OutputStream output) throws IOException; } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/NonClosingMigrationExportStream.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/NonClosingMigrationExportStream.java new file mode 100644 index 0000000000..41f65d4c4e --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/NonClosingMigrationExportStream.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Objects; + +/** Renderer-facing view that cannot close or clear its owner-managed buffer. */ +final class NonClosingMigrationExportStream extends OutputStream { + + private final OutputStream delegate; + + NonClosingMigrationExportStream(OutputStream delegate) { + this.delegate = Objects.requireNonNull(delegate, "delegate"); + } + + @Override + public void write(int value) throws IOException { + delegate.write(value); + } + + @Override + public void write(byte[] bytes, int offset, int length) throws IOException { + delegate.write(bytes, offset, length); + } + + @Override + public void flush() throws IOException { + delegate.flush(); + } + + @Override + public void close() { + // The renderer borrows this view; buffer ownership remains with PreparedMigrationExport. + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/PreparedMigrationExport.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/PreparedMigrationExport.java new file mode 100644 index 0000000000..1268ff7cb8 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/PreparedMigrationExport.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.IOException; +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportResponse; +import org.apache.hertzbeat.manager.setup.config.SecretValue; + +/** One-shot export preparation that owns only an independent clearable secret. */ +public final class PreparedMigrationExport implements AutoCloseable { + + static final int MAX_EXPORT_BYTES = 1024 * 1024; + + private final ExportResponse metadata; + private final SecretValue ownedSecret; + private final MigrationExportRenderer renderer; + private final BufferFactory bufferFactory; + private State state = State.READY; + + private PreparedMigrationExport( + ExportResponse metadata, SecretValue ownedSecret, MigrationExportRenderer renderer, + BufferFactory bufferFactory) { + this.metadata = Objects.requireNonNull(metadata, "metadata"); + this.ownedSecret = Objects.requireNonNull(ownedSecret, "ownedSecret"); + this.renderer = Objects.requireNonNull(renderer, "renderer"); + this.bufferFactory = Objects.requireNonNull(bufferFactory, "bufferFactory"); + } + + /** Copies the borrowed secret immediately; ownership of the caller's value never changes. */ + static PreparedMigrationExport prepare( + ExportResponse metadata, SecretValue borrowedSecret, MigrationExportRenderer renderer) { + Objects.requireNonNull(metadata, "metadata"); + Objects.requireNonNull(borrowedSecret, "borrowedSecret"); + Objects.requireNonNull(renderer, "renderer"); + return prepare(metadata, borrowedSecret, renderer, BoundedMigrationExportBuffer::new); + } + + static PreparedMigrationExport prepare( + ExportResponse metadata, SecretValue borrowedSecret, MigrationExportRenderer renderer, + BufferFactory bufferFactory) { + Objects.requireNonNull(metadata, "metadata"); + Objects.requireNonNull(borrowedSecret, "borrowedSecret"); + Objects.requireNonNull(renderer, "renderer"); + Objects.requireNonNull(bufferFactory, "bufferFactory"); + return new PreparedMigrationExport( + metadata, SecretValue.copyOf(borrowedSecret), renderer, bufferFactory); + } + + public ExportResponse metadata() { + return metadata; + } + + /** Renders and finalizes a bounded payload before any HTTP response is mutated. */ + public StagedMigrationExport stage() throws IOException { + beginStage(); + BoundedMigrationExportBuffer buffer = null; + boolean transferred = false; + try { + buffer = bufferFactory.create(MAX_EXPORT_BYTES); + renderer.write(ownedSecret, new NonClosingMigrationExportStream(buffer)); + StagedMigrationExport staged = buffer.finish(metadata); + transferred = true; + return staged; + } catch (IOException failure) { + throw new IOException("Prepared migration export failed"); + } catch (RuntimeException failure) { + throw new PreparedMigrationExportException(); + } finally { + if (!transferred && buffer != null) { + buffer.close(); + } + ownedSecret.close(); + finishStage(); + } + } + + @Override + public synchronized void close() { + if (state == State.CLOSED) { + return; + } + if (state == State.STAGING) { + throw new PreparedMigrationExportException(); + } + state = State.CLOSED; + ownedSecret.close(); + } + + @Override + public synchronized String toString() { + return "PreparedMigrationExport[state=" + state + "]"; + } + + private synchronized void beginStage() { + if (state != State.READY) { + throw new PreparedMigrationExportException(); + } + state = State.STAGING; + } + + private synchronized void finishStage() { + state = State.CLOSED; + } + + private enum State { + READY, + STAGING, + CLOSED + } + + @FunctionalInterface + interface BufferFactory { + BoundedMigrationExportBuffer create(int limit); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/PreparedMigrationExportException.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/PreparedMigrationExportException.java new file mode 100644 index 0000000000..4a7b5ccb2d --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/PreparedMigrationExportException.java @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +/** Cause-free lifecycle failure for a prepared export capability. */ +final class PreparedMigrationExportException extends IllegalStateException { + + PreparedMigrationExportException() { + super("Prepared migration export is unavailable"); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/StagedMigrationExport.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/StagedMigrationExport.java new file mode 100644 index 0000000000..f35aea6ec2 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/StagedMigrationExport.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportResponse; + +/** Finalized bounded export payload; closing clears every owned byte. */ +public final class StagedMigrationExport implements AutoCloseable { + + private final ExportResponse metadata; + private byte[] bytes; + private final int size; + private State state = State.READY; + + StagedMigrationExport(ExportResponse metadata, byte[] bytes, int size) { + this.metadata = Objects.requireNonNull(metadata, "metadata"); + this.bytes = Objects.requireNonNull(bytes, "bytes"); + if (size < 0 || size > bytes.length) { + throw new IllegalArgumentException("size"); + } + this.size = size; + } + + public ExportResponse metadata() { + return metadata; + } + + public int size() { + return size; + } + + public void writeTo(OutputStream output) throws IOException { + Objects.requireNonNull(output, "output"); + byte[] payload = beginWrite(); + try { + output.write(payload, 0, size); + } finally { + finishWrite(); + } + } + + @Override + public synchronized void close() { + if (state == State.CLOSED) { + return; + } + if (state == State.WRITING) { + throw new PreparedMigrationExportException(); + } + clear(); + } + + private synchronized byte[] beginWrite() { + if (state != State.READY) { + throw new PreparedMigrationExportException(); + } + state = State.WRITING; + return bytes; + } + + private synchronized void finishWrite() { + clear(); + } + + private void clear() { + Arrays.fill(bytes, (byte) 0); + bytes = new byte[0]; + state = State.CLOSED; + } + + private enum State { + READY, + WRITING, + CLOSED + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContractTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContractTest.java index 039994bbb8..0700e29237 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContractTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentApiContractTest.java @@ -45,6 +45,7 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseK import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreKind; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreSummary; +import org.apache.hertzbeat.manager.setup.workflow.PreparedMigrationExport; import org.junit.jupiter.api.Test; /** Freezes authenticated deployment and H2 migration contracts. */ @@ -69,7 +70,8 @@ class DeploymentApiContractTest { "maintenanceAdmission", "activeOperationId"); assertComponents(DeploymentApiContract.MetadataMigrationValidationRequest.class, "target", "targetDatabase"); - assertComponents(DeploymentApiContract.MetadataMigrationRequest.class, "target", "targetDatabase", "applyMode"); + assertComponents(DeploymentApiContract.MetadataMigrationRequest.class, + "operationId", "target", "targetDatabase", "applyMode"); assertComponents(DeploymentApiContract.MigrationView.class, "operationId", "state", "source", "target", "stage", "progressPercent", "createdAt", "startedAt", "completedAt", "verificationState", "errorCode", "nextPollAfterMillis", "activationAvailable", "restartRequired", @@ -96,6 +98,10 @@ class DeploymentApiContractTest { DeploymentWorkflow.class.getMethod( "validate", DeploymentApiContract.MetadataMigrationValidationRequest.class) .getReturnType()); + assertEquals(PreparedMigrationExport.class, + DeploymentWorkflow.class.getMethod( + "prepareExport", String.class, DeploymentApiContract.MigrationExportRequest.class) + .getReturnType()); } @Test @@ -183,15 +189,22 @@ class DeploymentApiContractTest { MetadataDatabaseConfiguration mysql = new MetadataDatabaseConfiguration( MetadataDatabaseKind.MYSQL, "jdbc:mysql://db/hertzbeat", "user", "secret"); assertEquals(mysql, new DeploymentApiContract.MetadataMigrationRequest( - MigrationTarget.MYSQL, mysql, ApplyMode.MANAGED_WRITE).targetDatabase()); + "migration-1", MigrationTarget.MYSQL, mysql, ApplyMode.MANAGED_WRITE).targetDatabase()); MetadataDatabaseConfiguration postgres = new MetadataDatabaseConfiguration( MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db/hertzbeat", "user", "secret"); assertThrows(IllegalArgumentException.class, () -> new DeploymentApiContract.MetadataMigrationRequest( - MigrationTarget.MYSQL, postgres, ApplyMode.MANAGED_WRITE)); + "migration-1", MigrationTarget.MYSQL, postgres, ApplyMode.MANAGED_WRITE)); MetadataDatabaseConfiguration h2 = new MetadataDatabaseConfiguration( MetadataDatabaseKind.H2, "jdbc:h2:file:./data/hertzbeat", "sa", "secret"); assertThrows(IllegalArgumentException.class, () -> new DeploymentApiContract.MetadataMigrationRequest( - MigrationTarget.POSTGRESQL, h2, ApplyMode.EXTERNAL_APPLY)); + "migration-1", MigrationTarget.POSTGRESQL, h2, ApplyMode.EXTERNAL_APPLY)); + + assertThrows(IllegalArgumentException.class, () -> new DeploymentApiContract.MetadataMigrationRequest( + null, MigrationTarget.MYSQL, mysql, ApplyMode.MANAGED_WRITE)); + assertThrows(IllegalArgumentException.class, () -> new DeploymentApiContract.MetadataMigrationRequest( + ".hidden", MigrationTarget.MYSQL, mysql, ApplyMode.MANAGED_WRITE)); + assertThrows(IllegalArgumentException.class, () -> new DeploymentApiContract.MetadataMigrationRequest( + "a".repeat(129), MigrationTarget.MYSQL, mysql, ApplyMode.MANAGED_WRITE)); } @Test @@ -200,13 +213,17 @@ class DeploymentApiContractTest { MetadataDatabaseConfiguration mysql = new MetadataDatabaseConfiguration( MetadataDatabaseKind.MYSQL, "jdbc:mysql://db/hertzbeat", "user", secret); DeploymentApiContract.MetadataMigrationRequest request = new DeploymentApiContract.MetadataMigrationRequest( - MigrationTarget.MYSQL, mysql, ApplyMode.MANAGED_WRITE); + "migration-1", MigrationTarget.MYSQL, mysql, ApplyMode.MANAGED_WRITE); assertFalse(objectMapper.writeValueAsString(request).contains(secret)); assertFalse(request.toString().contains(secret)); DeploymentApiContract.MigrationExportRequest export = new DeploymentApiContract.MigrationExportRequest( SetupApiContract.ExportFormat.ENV, MigrationOperationState.AWAITING_EXTERNAL_APPLY, mysql); assertFalse(objectMapper.writeValueAsString(export).contains(secret)); assertFalse(export.toString().contains(secret)); + assertThrows(IllegalArgumentException.class, + () -> new SetupApiContract.ExportResponse("unsafe\".env", "text/plain")); + assertThrows(IllegalArgumentException.class, + () -> new SetupApiContract.ExportResponse("safe.env", "text/plain\r\nprivate-header")); } @Test diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerExportTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerExportTest.java new file mode 100644 index 0000000000..f1d2ab5d75 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerExportTest.java @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.api; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.List; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationExportRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportFormat; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.workflow.PreparedMigrationExport; +import org.apache.hertzbeat.manager.setup.workflow.StagedMigrationExport; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.support.StaticListableBeanFactory; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; + +/** Bounded synchronous export and servlet failure contracts for deployment routes. */ +class DeploymentControllerExportTest { + + private final DeploymentWorkflow workflow = mock(DeploymentWorkflow.class); + private ObjectProvider workflowProvider; + private MockMvc mvc; + + @BeforeEach + void setUp() { + StaticListableBeanFactory factory = new StaticListableBeanFactory(); + factory.addBean("workflow", workflow); + workflowProvider = factory.getBeanProvider(DeploymentWorkflow.class); + mvc = MockMvcBuilders.standaloneSetup(new DeploymentController(workflowProvider)) + .setControllerAdvice(new SetupExceptionHandler()).build(); + } + + @Test + void externalApplyExportWritesOnlyAfterNoStoreAttachmentIsPrepared() throws Exception { + ExportFixture fixture = fixture(); + when(workflow.prepareExport(eq("migration-1"), any())).thenReturn(fixture.prepared); + String request = """ + {"format":"env","expectedState":"awaiting_external_apply", + "targetDatabase":{"kind":"mysql","jdbcUrl":"jdbc:mysql://db/hertzbeat", + "username":"operator","password":"export-secret"}} + """; + + mvc.perform(post(DeploymentApiContract.EXPORT_PATH, "migration-1") + .contentType(MediaType.APPLICATION_JSON).content(request)) + .andExpect(status().isOk()) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(header().string("Content-Disposition", + "attachment; filename=\"hertzbeat-migration.env\"")); + + verify(fixture.staged).writeTo(any()); + verify(fixture.staged).close(); + verify(fixture.prepared).close(); + } + + @Test + void exportBodyIsWrittenSynchronouslyWithoutCapturingTheRequest() throws Exception { + ExportFixture fixture = fixture(); + + fixture.controller.export("migration-1", fixture.request, fixture.response); + + verify(fixture.prepared).stage(); + verify(fixture.staged).metadata(); + verify(fixture.staged).writeTo(fixture.output); + verify(fixture.response).flushBuffer(); + verify(fixture.staged).close(); + verify(fixture.prepared).close(); + } + + @Test + void renderFailureClosesPreparedWithoutMutatingTheServletResponse() throws Exception { + MigrationExportRequest request = exportRequest(); + PreparedMigrationExport prepared = mock(PreparedMigrationExport.class); + when(workflow.prepareExport("migration-1", request)).thenReturn(prepared); + when(prepared.stage()).thenThrow(new IOException("Prepared migration export failed")); + HttpServletResponse response = mock(HttpServletResponse.class); + DeploymentController controller = new DeploymentController(workflowProvider); + + assertThatThrownBy(() -> controller.export("migration-1", request, response)) + .isInstanceOf(IOException.class) + .hasMessageNotContaining("export-secret"); + + verify(prepared).close(); + verifyNoInteractions(response); + } + + @Test + void outputFailureClosesPreparedExportAndResetsAnUncommittedResponse() throws Exception { + ExportFixture fixture = fixture(); + IOException outputFailure = new IOException("private-output-detail"); + when(fixture.response.isCommitted()).thenReturn(false); + org.mockito.Mockito.doThrow(outputFailure).when(fixture.staged).writeTo(fixture.output); + + assertThatThrownBy(() -> fixture.controller.export( + "migration-1", fixture.request, fixture.response)).isSameAs(outputFailure); + + verify(fixture.response).reset(); + verify(fixture.staged).close(); + verify(fixture.prepared).close(); + } + + @Test + void committedTransportFailureDoesNotAttemptToResetTheResponse() throws Exception { + ExportFixture fixture = fixture(); + IOException outputFailure = new IOException("private-output-detail"); + when(fixture.response.isCommitted()).thenReturn(true); + org.mockito.Mockito.doThrow(outputFailure).when(fixture.staged).writeTo(fixture.output); + + assertThatThrownBy(() -> fixture.controller.export( + "migration-1", fixture.request, fixture.response)).isSameAs(outputFailure); + + verify(fixture.response, never()).reset(); + verify(fixture.staged).close(); + verify(fixture.prepared).close(); + } + + @Test + void ordinaryResponseRuntimeAtEveryMutationBoundaryResetsWithoutReplacingFailure() throws Exception { + for (ResponseFailurePoint failurePoint : ResponseFailurePoint.values()) { + ExportFixture fixture = fixture(); + when(fixture.response.isCommitted()).thenReturn(false); + IllegalStateException failure = new IllegalStateException("private-response-runtime"); + failurePoint.fail(fixture, failure); + + assertThatThrownBy(() -> fixture.controller.export( + "migration-1", fixture.request, fixture.response)).isSameAs(failure); + + verify(fixture.response).reset(); + verify(fixture.staged).close(); + verify(fixture.prepared).close(); + } + } + + @Test + void rollbackRuntimeNeverReplacesTheOriginalResponseFailure() throws Exception { + for (boolean failCommittedCheck : List.of(true, false)) { + ExportFixture fixture = fixture(); + IllegalStateException original = new IllegalStateException("original-response-failure"); + org.mockito.Mockito.doThrow(original).when(fixture.staged).writeTo(fixture.output); + if (failCommittedCheck) { + when(fixture.response.isCommitted()) + .thenThrow(new IllegalStateException("private-committed-failure")); + } else { + when(fixture.response.isCommitted()).thenReturn(false); + org.mockito.Mockito.doThrow(new IllegalStateException("private-reset-failure")) + .when(fixture.response).reset(); + } + + assertThatThrownBy(() -> fixture.controller.export( + "migration-1", fixture.request, fixture.response)).isSameAs(original); + } + } + + @Test + void fatalResponseErrorRemainsRawAndPrimary() throws Exception { + ExportFixture fixture = fixture(); + AssertionError fatal = new AssertionError("response-fatal"); + org.mockito.Mockito.doThrow(fatal).when(fixture.response) + .setHeader("Cache-Control", "no-store"); + + assertThatThrownBy(() -> fixture.controller.export( + "migration-1", fixture.request, fixture.response)).isSameAs(fatal); + + verify(fixture.response, never()).isCommitted(); + verify(fixture.response, never()).reset(); + verify(fixture.staged).close(); + verify(fixture.prepared).close(); + } + + private ExportFixture fixture() throws Exception { + MigrationExportRequest request = exportRequest(); + PreparedMigrationExport prepared = mock(PreparedMigrationExport.class); + StagedMigrationExport staged = mock(StagedMigrationExport.class); + HttpServletResponse response = mock(HttpServletResponse.class); + ServletOutputStream output = mock(ServletOutputStream.class); + when(prepared.stage()).thenReturn(staged); + when(staged.metadata()).thenReturn( + new ExportResponse("hertzbeat-migration.env", "text/plain")); + when(staged.size()).thenReturn(8); + when(workflow.prepareExport("migration-1", request)).thenReturn(prepared); + when(response.getOutputStream()).thenReturn(output); + return new ExportFixture(request, prepared, staged, response, output, + new DeploymentController(workflowProvider)); + } + + private MigrationExportRequest exportRequest() { + return new MigrationExportRequest(ExportFormat.ENV, + MigrationOperationState.AWAITING_EXTERNAL_APPLY, + new MetadataDatabaseConfiguration(MetadataDatabaseKind.MYSQL, + "jdbc:mysql://db/hertzbeat", "operator", "export-secret")); + } + + private record ExportFixture( + MigrationExportRequest request, + PreparedMigrationExport prepared, + StagedMigrationExport staged, + HttpServletResponse response, + ServletOutputStream output, + DeploymentController controller) { } + + private enum ResponseFailurePoint { + HEADER { + @Override + void fail(ExportFixture fixture, RuntimeException failure) { + org.mockito.Mockito.doThrow(failure).when(fixture.response) + .setHeader("Cache-Control", "no-store"); + } + }, + OUTPUT_STREAM { + @Override + void fail(ExportFixture fixture, RuntimeException failure) throws IOException { + when(fixture.response.getOutputStream()).thenThrow(failure); + } + }, + WRITE { + @Override + void fail(ExportFixture fixture, RuntimeException failure) throws IOException { + org.mockito.Mockito.doThrow(failure).when(fixture.staged).writeTo(fixture.output); + } + }; + + abstract void fail(ExportFixture fixture, RuntimeException failure) throws IOException; + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerRegistrationTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerRegistrationTest.java index f3a35c42ad..f8259aaa9f 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerRegistrationTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerRegistrationTest.java @@ -22,7 +22,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; -import org.apache.hertzbeat.manager.setup.workflow.MigrationExportRenderer; import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.runner.ApplicationContextRunner; import org.springframework.context.annotation.ComponentScan; @@ -48,9 +47,8 @@ class DeploymentControllerRegistrationTest { } @Test - void componentScanResolvesWorkflowAndRendererRegisteredAlongsideController() { + void componentScanResolvesWorkflowRegisteredAlongsideController() { context.withBean(DeploymentWorkflow.class, () -> mock(DeploymentWorkflow.class)) - .withBean(MigrationExportRenderer.class, () -> mock(MigrationExportRenderer.class)) .run(result -> assertThat(result).hasSingleBean(DeploymentController.class)); } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerTest.java index 3a97b5cd79..0ecc331cac 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/DeploymentControllerTest.java @@ -18,21 +18,19 @@ package org.apache.hertzbeat.manager.setup.api; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.asyncDispatch; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.request; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import java.io.ByteArrayOutputStream; import java.time.Instant; import java.util.List; import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionException; @@ -41,49 +39,38 @@ import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentVi import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MaintenanceAdmission; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MaintenanceMode; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationCapability; -import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationExportRequest; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationView; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportFormat; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigSource; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ManagementDatabaseSummary; -import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationResponse; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreKind; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreSummary; -import org.apache.hertzbeat.manager.setup.workflow.MigrationExportRenderer; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.support.StaticListableBeanFactory; import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; import org.springframework.test.web.servlet.MockMvc; -import org.springframework.test.web.servlet.MvcResult; import org.springframework.test.web.servlet.setup.MockMvcBuilders; -import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; /** Transport proof for authenticated, no-store deployment routes. */ class DeploymentControllerTest { private final DeploymentWorkflow workflow = mock(DeploymentWorkflow.class); - private final MigrationExportRenderer exportRenderer = mock(MigrationExportRenderer.class); private ObjectProvider workflowProvider; - private ObjectProvider rendererProvider; private MockMvc mvc; @BeforeEach void setUp() { - StaticListableBeanFactory factory = providerFactory(List.of(workflow), List.of(exportRenderer)); + StaticListableBeanFactory factory = providerFactory(List.of(workflow)); workflowProvider = factory.getBeanProvider(DeploymentWorkflow.class); - rendererProvider = factory.getBeanProvider(MigrationExportRenderer.class); - mvc = mvc(workflowProvider, rendererProvider); + mvc = mvc(workflowProvider); } @Test @@ -99,7 +86,7 @@ class DeploymentControllerTest { "jdbcUrl":"jdbc:mysql://db/hertzbeat","username":"operator","password":"request-secret"}} """; String migration = """ - {"target":"mysql","targetDatabase":{"kind":"mysql", + {"operationId":"migration-1","target":"mysql","targetDatabase":{"kind":"mysql", "jdbcUrl":"jdbc:mysql://db/hertzbeat","username":"operator","password":"request-secret"}, "applyMode":"managed_write"} """; @@ -130,6 +117,7 @@ class DeploymentControllerTest { .andExpect(jsonPath("$.restartRequired").value(true)); verify(workflow).migration("migration-1"); + verify(workflow).migrate(argThat(request -> "migration-1".equals(request.operationId()))); } @Test @@ -171,7 +159,27 @@ class DeploymentControllerTest { } @Test - void rejectsInvalidOperationIdsBeforeWorkflowOrRendererDispatch() throws Exception { + void rejectsInvalidOperationIdsBeforeWorkflowDispatch() throws Exception { + String migrationWithoutId = """ + {"target":"mysql","targetDatabase":{"kind":"mysql", + "jdbcUrl":"jdbc:mysql://db/hertzbeat","username":"operator","password":"request-secret"}, + "applyMode":"managed_write"} + """; + String migrationWithUnsafeId = """ + {"operationId":".hidden","target":"mysql","targetDatabase":{"kind":"mysql", + "jdbcUrl":"jdbc:mysql://db/hertzbeat","username":"operator","password":"request-secret"}, + "applyMode":"managed_write"} + """; + mvc.perform(post(DeploymentApiContract.MIGRATION_PATH).contentType(MediaType.APPLICATION_JSON) + .content(migrationWithoutId)) + .andExpect(status().isBadRequest()) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.errorCode").value("invalid_request")); + mvc.perform(post(DeploymentApiContract.MIGRATION_PATH).contentType(MediaType.APPLICATION_JSON) + .content(migrationWithUnsafeId)) + .andExpect(status().isBadRequest()) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(jsonPath("$.errorCode").value("invalid_request")); mvc.perform(get(DeploymentApiContract.MIGRATION_OPERATION_PATH, ".hidden")) .andExpect(status().isBadRequest()) .andExpect(header().string("Cache-Control", "no-store")) @@ -193,63 +201,17 @@ class DeploymentControllerTest { .andExpect(header().string("Cache-Control", "no-store")) .andExpect(jsonPath("$.errorCode").value("invalid_request")); - verifyNoInteractions(workflow, exportRenderer); + verifyNoInteractions(workflow); } @Test void missingWorkflowReturnsStableNoStoreUnavailable() throws Exception { - assertDeploymentUnavailable(mvc(List.of(), List.of(exportRenderer))); + assertDeploymentUnavailable(mvc(List.of())); } @Test void ambiguousWorkflowReturnsStableNoStoreUnavailable() throws Exception { - assertDeploymentUnavailable(mvc( - List.of(workflow, mock(DeploymentWorkflow.class)), List.of(exportRenderer))); - } - - @Test - void missingOrAmbiguousRendererReturnsStableNoStoreUnavailable() throws Exception { - assertExportUnavailable(mvc(List.of(workflow), List.of())); - assertExportUnavailable(mvc(List.of(workflow), - List.of(exportRenderer, mock(MigrationExportRenderer.class)))); - } - - @Test - void externalApplyExportStreamsOnlyAfterNoStoreAttachmentIsPrepared() throws Exception { - when(workflow.prepareExport(eq("migration-1"), any())).thenReturn( - new ExportResponse("hertzbeat-migration.env", "text/plain")); - String request = """ - {"format":"env","expectedState":"awaiting_external_apply", - "targetDatabase":{"kind":"mysql","jdbcUrl":"jdbc:mysql://db/hertzbeat", - "username":"operator","password":"export-secret"}} - """; - - MvcResult pending = mvc.perform(post(DeploymentApiContract.EXPORT_PATH, "migration-1") - .contentType(MediaType.APPLICATION_JSON).content(request)) - .andExpect(request().asyncStarted()).andReturn(); - mvc.perform(asyncDispatch(pending)) - .andExpect(status().isOk()) - .andExpect(header().string("Cache-Control", "no-store")) - .andExpect(header().string("Content-Disposition", - "attachment; filename=\"hertzbeat-migration.env\"")); - verify(exportRenderer).write(eq("migration-1"), any(), any()); - } - - @Test - void exportBodyIsDeferredUntilTheStreamingCallbackRuns() throws Exception { - MigrationExportRequest request = new MigrationExportRequest(ExportFormat.ENV, - MigrationOperationState.AWAITING_EXTERNAL_APPLY, - new MetadataDatabaseConfiguration(MetadataDatabaseKind.MYSQL, - "jdbc:mysql://db/hertzbeat", "operator", "export-secret")); - when(workflow.prepareExport("migration-1", request)).thenReturn( - new ExportResponse("hertzbeat-migration.env", "text/plain")); - DeploymentController controller = new DeploymentController(workflowProvider, rendererProvider); - - ResponseEntity response = controller.export("migration-1", request); - - verifyNoInteractions(exportRenderer); - response.getBody().writeTo(new ByteArrayOutputStream()); - verify(exportRenderer).write(eq("migration-1"), eq(request), any()); + assertDeploymentUnavailable(mvc(List.of(workflow, mock(DeploymentWorkflow.class)))); } private MigrationView readyMigration() { @@ -277,29 +239,21 @@ class DeploymentControllerTest { VerificationState.SUCCEEDED, null, 1000, false, true, false); } - private MockMvc mvc( - List workflows, List renderers) { - StaticListableBeanFactory factory = providerFactory(workflows, renderers); - return mvc(factory.getBeanProvider(DeploymentWorkflow.class), - factory.getBeanProvider(MigrationExportRenderer.class)); + private MockMvc mvc(List workflows) { + StaticListableBeanFactory factory = providerFactory(workflows); + return mvc(factory.getBeanProvider(DeploymentWorkflow.class)); } - private MockMvc mvc( - ObjectProvider workflows, - ObjectProvider renderers) { - return MockMvcBuilders.standaloneSetup(new DeploymentController(workflows, renderers)) + private MockMvc mvc(ObjectProvider workflows) { + return MockMvcBuilders.standaloneSetup(new DeploymentController(workflows)) .setControllerAdvice(new SetupExceptionHandler()).build(); } - private StaticListableBeanFactory providerFactory( - List workflows, List renderers) { + private StaticListableBeanFactory providerFactory(List workflows) { StaticListableBeanFactory factory = new StaticListableBeanFactory(); for (int index = 0; index < workflows.size(); index++) { factory.addBean("workflow-" + index, workflows.get(index)); } - for (int index = 0; index < renderers.size(); index++) { - factory.addBean("renderer-" + index, renderers.get(index)); - } return factory; } @@ -310,16 +264,4 @@ class DeploymentControllerTest { .andExpect(jsonPath("$.errorCode").value("migration_unavailable")); } - private void assertExportUnavailable(MockMvc candidate) throws Exception { - String request = """ - {"format":"env","expectedState":"awaiting_external_apply", - "targetDatabase":{"kind":"mysql","jdbcUrl":"jdbc:mysql://db/hertzbeat", - "username":"operator","password":"export-secret"}} - """; - candidate.perform(post(DeploymentApiContract.EXPORT_PATH, "migration-1") - .contentType(MediaType.APPLICATION_JSON).content(request)) - .andExpect(status().isServiceUnavailable()) - .andExpect(header().string("Cache-Control", "no-store")) - .andExpect(jsonPath("$.errorCode").value("migration_unavailable")); - } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java index 421003bd2a..f37c0b58ef 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/api/SetupApiContractTest.java @@ -222,8 +222,9 @@ class SetupApiContractTest { "public_address_invalid", "mail_connection_failed", "administrator_already_configured", "administrator_username_invalid", "operation_not_found", "operation_conflict", "migration_source_unsupported", "migration_target_not_empty", "migration_multi_node_unsupported", - "migration_copy_failed", "migration_verification_failed", "migration_activation_failed", - "restart_failed"); + "migration_topology_unavailable", "migration_maintenance_required", "migration_unavailable", + "migration_copy_failed", "migration_verification_failed", "migration_activation_not_available", + "migration_activation_failed", "restart_failed"); } @Test diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/BoundedMigrationExportBufferTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/BoundedMigrationExportBufferTest.java new file mode 100644 index 0000000000..d94d389847 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/BoundedMigrationExportBufferTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportResponse; +import org.junit.jupiter.api.Test; + +/** Exact byte ownership contracts for finalized migration export staging. */ +class BoundedMigrationExportBufferTest { + + @Test + void finalPayloadConstructionErrorClearsTheStillOwnedBytes() throws Exception { + AssertionError fatal = new AssertionError("staged-construction-fatal"); + AtomicReference attemptedBytes = new AtomicReference<>(); + BoundedMigrationExportBuffer buffer = new BoundedMigrationExportBuffer(64, + (metadata, bytes, size) -> { + attemptedBytes.set(bytes); + throw fatal; + }); + buffer.write("sensitive-payload".getBytes(StandardCharsets.UTF_8)); + + assertThatThrownBy(() -> buffer.finish(new ExportResponse("export.env", "text/plain"))) + .isSameAs(fatal); + + assertThat(attemptedBytes.get()).containsOnly((byte) 0); + } + + @Test + void ownerCloseIsTerminalAndCannotRegrowOrFinalizeTheBuffer() throws Exception { + BoundedMigrationExportBuffer buffer = new BoundedMigrationExportBuffer(64); + buffer.write("sensitive-payload".getBytes(StandardCharsets.UTF_8)); + + buffer.close(); + buffer.close(); + + assertThatThrownBy(() -> buffer.write(1)).isInstanceOf(IOException.class); + assertThatThrownBy(() -> buffer.finish(new ExportResponse("export.env", "text/plain"))) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/PreparedMigrationExportTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/PreparedMigrationExportTest.java new file mode 100644 index 0000000000..291276a4a8 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/PreparedMigrationExportTest.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertSame; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ExportResponse; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.junit.jupiter.api.Test; + +/** Ownership, bounded staging, and one-shot lifecycle contracts for migration export. */ +class PreparedMigrationExportTest { + + private static final ExportResponse METADATA = + new ExportResponse("hertzbeat-migration.env", "text/plain"); + + @Test + void stagesIntoOwnedBytesBeforeWritingAndClearsItsIndependentSecret() throws Exception { + SecretValue borrowed = SecretValue.of("export-secret"); + AtomicReference rendererSecret = new AtomicReference<>(); + MigrationExportRenderer renderer = (secret, output) -> { + rendererSecret.set(secret); + output.write("rendered".getBytes(StandardCharsets.UTF_8)); + }; + PreparedMigrationExport prepared = PreparedMigrationExport.prepare(METADATA, borrowed, renderer); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + try (StagedMigrationExport staged = prepared.stage()) { + assertSame(METADATA, staged.metadata()); + assertThat(staged.size()).isEqualTo(8); + assertThat(output.size()).isZero(); + staged.writeTo(output); + } + + assertThat(output.toString(StandardCharsets.UTF_8)).isEqualTo("rendered"); + assertThat(rendererSecret.get()).isNotSameAs(borrowed); + assertThat(rendererSecret.get().copy()).containsOnly('\0'); + assertThat(borrowed.copy()).containsExactly("export-secret".toCharArray()); + prepared.close(); + borrowed.close(); + } + + @Test + void rendererFailureProducesNoStagedPayloadAndClearsSecret() { + SecretValue borrowed = SecretValue.of("failure-secret"); + AtomicReference rendererSecret = new AtomicReference<>(); + PreparedMigrationExport prepared = PreparedMigrationExport.prepare(METADATA, borrowed, + (secret, output) -> { + rendererSecret.set(secret); + output.write("partial-private".getBytes(StandardCharsets.UTF_8)); + throw new IOException("private-renderer-detail"); + }); + + assertThatThrownBy(prepared::stage) + .isInstanceOf(IOException.class) + .hasMessageNotContaining("private-renderer-detail") + .hasNoCause(); + assertThat(rendererSecret.get().copy()).containsOnly('\0'); + assertThat(borrowed.copy()).containsExactly("failure-secret".toCharArray()); + borrowed.close(); + } + + @Test + void rejectsPayloadPastTheFixedLimitWithoutReturningPartialBytes() { + SecretValue borrowed = SecretValue.of("bounded-secret"); + PreparedMigrationExport prepared = PreparedMigrationExport.prepare(METADATA, borrowed, + (secret, output) -> output.write(new byte[PreparedMigrationExport.MAX_EXPORT_BYTES + 1])); + + assertThatThrownBy(prepared::stage) + .isInstanceOf(IOException.class) + .hasNoCause() + .hasMessageNotContaining("bounded-secret"); + borrowed.close(); + } + + @Test + void rendererCloseAtEndDoesNotDiscardThePreparedPayload() throws Exception { + SecretValue borrowed = SecretValue.of("close-at-end-secret"); + PreparedMigrationExport prepared = PreparedMigrationExport.prepare(METADATA, borrowed, + (secret, output) -> { + output.write("complete".getBytes(StandardCharsets.UTF_8)); + output.close(); + }); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + try (StagedMigrationExport staged = prepared.stage()) { + staged.writeTo(output); + } + + assertThat(output.toString(StandardCharsets.UTF_8)).isEqualTo("complete"); + borrowed.close(); + } + + @Test + void rendererCloseAndContinuedWriteRemainValid() throws Exception { + SecretValue borrowed = SecretValue.of("close-then-write-secret"); + PreparedMigrationExport prepared = PreparedMigrationExport.prepare(METADATA, borrowed, + (secret, output) -> { + output.close(); + output.write("after-close".getBytes(StandardCharsets.UTF_8)); + }); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + try (StagedMigrationExport staged = prepared.stage()) { + staged.writeTo(output); + } + + assertThat(output.toString(StandardCharsets.UTF_8)).isEqualTo("after-close"); + borrowed.close(); + } + + @Test + void closeBeforeStageClearsOwnedSecretAndRejectsLaterConsumption() { + SecretValue borrowed = SecretValue.of("unused-secret"); + AtomicReference rendererSecret = new AtomicReference<>(); + PreparedMigrationExport prepared = PreparedMigrationExport.prepare(METADATA, borrowed, + (secret, output) -> rendererSecret.set(secret)); + + prepared.close(); + prepared.close(); + + assertThatThrownBy(prepared::stage) + .isInstanceOf(IllegalStateException.class) + .hasMessageNotContaining("unused-secret"); + assertThat(rendererSecret).hasValue(null); + borrowed.close(); + } + + @Test + void metadataAndDiagnosticsNeverExposeRendererOrSecret() { + SecretValue borrowed = SecretValue.of("diagnostic-secret"); + MigrationExportRenderer renderer = (secret, output) -> { }; + PreparedMigrationExport prepared = PreparedMigrationExport.prepare(METADATA, borrowed, renderer); + + assertSame(METADATA, prepared.metadata()); + assertThat(prepared.toString()) + .doesNotContain("diagnostic-secret") + .doesNotContain(renderer.getClass().getName()); + prepared.close(); + borrowed.close(); + } + + @Test + void firstRendererErrorRemainsPrimaryWhileOwnedMaterialIsCleared() { + SecretValue borrowed = SecretValue.of("fatal-secret"); + AtomicReference rendererSecret = new AtomicReference<>(); + AssertionError fatal = new AssertionError("renderer-fatal"); + PreparedMigrationExport prepared = PreparedMigrationExport.prepare(METADATA, borrowed, + (secret, output) -> { + rendererSecret.set(secret); + throw fatal; + }); + + assertThatThrownBy(prepared::stage).isSameAs(fatal).hasNoCause(); + assertThat(rendererSecret.get().copy()).containsOnly('\0'); + borrowed.close(); + } + + @Test + void stagedBytesAreClearedOnCloseAndCannotBeWrittenAgain() throws Exception { + SecretValue borrowed = SecretValue.of("staged-secret"); + PreparedMigrationExport prepared = PreparedMigrationExport.prepare(METADATA, borrowed, + (secret, output) -> output.write("payload".getBytes(StandardCharsets.UTF_8))); + StagedMigrationExport staged = prepared.stage(); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + staged.close(); + staged.close(); + + assertThatThrownBy(() -> staged.writeTo(output)) + .isInstanceOf(IllegalStateException.class) + .hasNoCause(); + assertThat(output.size()).isZero(); + borrowed.close(); + } + + @Test + void bufferAllocationErrorStillClearsOwnedSecretAndEndsPreparation() throws Exception { + SecretValue borrowed = SecretValue.of("allocation-secret"); + AssertionError fatal = new AssertionError("buffer-allocation-fatal"); + PreparedMigrationExport prepared = PreparedMigrationExport.prepare(METADATA, borrowed, + (secret, output) -> { }, limit -> { + throw fatal; + }); + Field secretField = PreparedMigrationExport.class.getDeclaredField("ownedSecret"); + secretField.setAccessible(true); + SecretValue ownedSecret = (SecretValue) secretField.get(prepared); + + assertThatThrownBy(prepared::stage).isSameAs(fatal); + + assertThat(ownedSecret.copy()).containsOnly('\0'); + assertThat(prepared.toString()).contains("CLOSED").doesNotContain("allocation-secret"); + prepared.close(); + borrowed.close(); + } +} From cc95d4a9ad5d9ee9e8d9d94c489d861993ac12fa Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 19:32:39 +0800 Subject: [PATCH 65/71] Record durable migration execution failures --- .../DurableCutoverFailureFinalizer.java | 63 +++++ .../setup/workflow/DurableKnownFailure.java | 50 ++++ .../workflow/FileMigrationOperationStore.java | 38 +++ .../DurableCutoverFailureFinalizerTest.java | 261 ++++++++++++++++++ 4 files changed, 412 insertions(+) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverFailureFinalizer.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableKnownFailure.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverFailureFinalizerTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverFailureFinalizer.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverFailureFinalizer.java new file mode 100644 index 0000000000..aade95e8db --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverFailureFinalizer.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.time.Instant; +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.workflow.FileMigrationOperationStore.ExactTransitionDisposition; + +/** Durably records only migration failures whose resource cleanup is already known to be complete. */ +final class DurableCutoverFailureFinalizer { + + private final FileMigrationOperationStore store; + + DurableCutoverFailureFinalizer(FileMigrationOperationStore store) { + this.store = Objects.requireNonNull(store, "store"); + } + + Disposition finalizeFailure( + String operationId, DurableKnownFailure failure, Instant completedAt) { + Objects.requireNonNull(failure, "failure"); + Objects.requireNonNull(completedAt, "completedAt"); + ExactTransitionDisposition result = store.transformAndTransitionOrConfirmDisposition( + operationId, current -> replacement(current, failure, completedAt)); + return result == ExactTransitionDisposition.TRANSITIONED + ? Disposition.TRANSITIONED : Disposition.ALREADY_CONFIRMED; + } + + private MigrationOperationSnapshot replacement( + MigrationOperationSnapshot current, DurableKnownFailure failure, Instant completedAt) { + if (current.state() == MigrationOperationState.FAILED) { + if (current.errorCode() == failure.errorCode() && current.completedAt().equals(completedAt)) { + return current; + } + throw conflict(); + } + if (current.state() != MigrationOperationState.RUNNING + || current.stage() != failure.requiredStage()) { + throw conflict(); + } + if (completedAt.isBefore(current.startedAt())) { + throw new IllegalArgumentException("Invalid migration completion time"); + } + return new MigrationOperationSnapshot( + current.operationId(), MigrationOperationState.FAILED, current.target(), current.applyMode(), + MigrationStage.FAILED, failure.progress(current), current.createdAt(), current.startedAt(), + completedAt, failure.verificationState(), failure.errorCode(), null, 0, + false, false, false, current.targetIdentityHash(), current.managedCandidateGeneration()); + } + + private static MigrationOperationStoreException conflict() { + return new MigrationOperationStoreException(SetupErrorCode.OPERATION_CONFLICT); + } + + enum Disposition { TRANSITIONED, ALREADY_CONFIRMED } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableKnownFailure.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableKnownFailure.java new file mode 100644 index 0000000000..a4066cea9b --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableKnownFailure.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Closed capability for failure outcomes whose migration resources are already known to be clean. */ +enum DurableKnownFailure { + COPY(SetupErrorCode.MIGRATION_COPY_FAILED, MigrationStage.COPYING, VerificationState.PENDING), + VERIFICATION( + SetupErrorCode.MIGRATION_VERIFICATION_FAILED, + MigrationStage.VERIFYING, + VerificationState.FAILED); + + private final SetupErrorCode errorCode; + private final MigrationStage requiredStage; + private final VerificationState verificationState; + + DurableKnownFailure( + SetupErrorCode errorCode, + MigrationStage requiredStage, + VerificationState verificationState) { + this.errorCode = errorCode; + this.requiredStage = requiredStage; + this.verificationState = verificationState; + } + + SetupErrorCode errorCode() { + return errorCode; + } + + MigrationStage requiredStage() { + return requiredStage; + } + + VerificationState verificationState() { + return verificationState; + } + + int progress(MigrationOperationSnapshot current) { + return this == COPY ? current.progressPercent() : 100; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java index 71a2178f6b..8fb01f4a08 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java @@ -158,6 +158,14 @@ public final class FileMigrationOperationStore implements MigrationOperationStor return locked(() -> transitionOrConfirm(read(), operationId, expectedState, replacement)); } + /** Derives and durably publishes one exact replacement while holding the store lock. */ + ExactTransitionDisposition transformAndTransitionOrConfirmDisposition( + String operationId, SnapshotTransition transition) { + requireSafeId(operationId); + Objects.requireNonNull(transition, "transition"); + return locked(() -> transformOrConfirm(read(), operationId, transition)); + } + private MigrationOperationSnapshot transition( List snapshots, String operationId, MigrationOperationState expectedState, MigrationOperationSnapshot replacement) { @@ -200,6 +208,30 @@ public final class FileMigrationOperationStore implements MigrationOperationStor throw failure(SetupErrorCode.OPERATION_NOT_FOUND); } + private ExactTransitionDisposition transformOrConfirm( + List snapshots, String operationId, SnapshotTransition transition) { + for (int index = 0; index < snapshots.size(); index++) { + MigrationOperationSnapshot current = snapshots.get(index); + if (current.operationId().equals(operationId)) { + MigrationOperationSnapshot replacement = Objects.requireNonNull( + transition.apply(current), "replacement"); + if (!replacement.operationId().equals(operationId)) { + throw failure(SetupErrorCode.OPERATION_CONFLICT); + } + if (current.equals(replacement)) { + writeAndConfirm(snapshots); + return ExactTransitionDisposition.ALREADY_CONFIRMED; + } + transitionPolicy.requireAllowed(current, replacement); + snapshots.set(index, replacement); + trim(snapshots); + writeAndConfirm(snapshots); + return ExactTransitionDisposition.TRANSITIONED; + } + } + throw failure(SetupErrorCode.OPERATION_NOT_FOUND); + } + private List read() { if (!Files.exists(operationFile, LinkOption.NOFOLLOW_LINKS)) { return new ArrayList<>(); @@ -299,5 +331,11 @@ public final class FileMigrationOperationStore implements MigrationOperationStor void publish(Path target, byte[] content) throws IOException; } + @FunctionalInterface + interface SnapshotTransition { + /** Pure in-memory transform; called while the operation-store lock is held. */ + MigrationOperationSnapshot apply(MigrationOperationSnapshot current); + } + enum ExactTransitionDisposition { TRANSITIONED, ALREADY_CONFIRMED } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverFailureFinalizerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverFailureFinalizerTest.java new file mode 100644 index 0000000000..791d591160 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverFailureFinalizerTest.java @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.nio.file.Path; +import java.time.Instant; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.security.CommittedSetupFileDurabilityException; +import org.apache.hertzbeat.manager.setup.workflow.DurableCutoverFailureFinalizer.Disposition; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; + +class DurableCutoverFailureFinalizerTest { + + private static final String OPERATION_ID = "operation-a"; + private static final String IDENTITY = "a".repeat(64); + private static final String GENERATION = "candidate-generation"; + private static final Instant CREATED = Instant.parse("2026-08-10T01:00:00Z"); + private static final Instant STARTED = CREATED.plusSeconds(1); + private static final Instant COMPLETED = STARTED.plusSeconds(20); + + @TempDir + private Path root; + + @Test + void terminalizesKnownCopyFailureWhilePreservingObservedProgressAndIdentity() { + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + MigrationOperationSnapshot copying = copying(37); + store.create(pending()); + store.compareAndTransition(OPERATION_ID, MigrationOperationState.PENDING, copying); + + Disposition result = new DurableCutoverFailureFinalizer(store).finalizeFailure( + OPERATION_ID, DurableKnownFailure.COPY, COMPLETED); + + assertThat(result).isEqualTo(Disposition.TRANSITIONED); + MigrationOperationSnapshot failed = store.find(OPERATION_ID).orElseThrow(); + assertThat(failed.state()).isEqualTo(MigrationOperationState.FAILED); + assertThat(failed.stage()).isEqualTo(MigrationStage.FAILED); + assertThat(failed.progressPercent()).isEqualTo(37); + assertThat(failed.verificationState()).isEqualTo(VerificationState.PENDING); + assertThat(failed.errorCode()).isEqualTo(SetupErrorCode.MIGRATION_COPY_FAILED); + assertThat(failed.completedAt()).isEqualTo(COMPLETED); + assertThat(failed.targetIdentityHash()).isEqualTo(IDENTITY); + assertThat(failed.managedCandidateGeneration()).isEqualTo(GENERATION); + } + + @Test + void terminalizesKnownVerificationFailureAtCompletedVerificationProgress() { + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + MigrationOperationSnapshot verifying = verifying(); + store.create(pending()); + store.compareAndTransition(OPERATION_ID, MigrationOperationState.PENDING, copying(40)); + store.compareAndTransition(OPERATION_ID, MigrationOperationState.RUNNING, verifying); + + Disposition result = new DurableCutoverFailureFinalizer(store).finalizeFailure( + OPERATION_ID, DurableKnownFailure.VERIFICATION, COMPLETED); + + assertThat(result).isEqualTo(Disposition.TRANSITIONED); + MigrationOperationSnapshot failed = store.find(OPERATION_ID).orElseThrow(); + assertThat(failed.state()).isEqualTo(MigrationOperationState.FAILED); + assertThat(failed.progressPercent()).isEqualTo(100); + assertThat(failed.verificationState()).isEqualTo(VerificationState.FAILED); + assertThat(failed.errorCode()).isEqualTo(SetupErrorCode.MIGRATION_VERIFICATION_FAILED); + } + + @Test + void acceptsOnlyTheClosedKnownCleanupFailureCapability() { + assertThat(DurableKnownFailure.values()) + .containsExactly(DurableKnownFailure.COPY, DurableKnownFailure.VERIFICATION); + assertThat(DurableCutoverFailureFinalizer.class.getDeclaredMethods()) + .filteredOn(method -> method.getName().equals("finalizeFailure")) + .singleElement() + .satisfies(method -> assertThat(method.getParameterTypes()) + .containsExactly(String.class, DurableKnownFailure.class, Instant.class) + .doesNotContain(SetupErrorCode.class, Throwable.class)); + } + + @Test + void rejectsWrongPhaseAndInvalidCompletionTimeWithoutChangingTheJournal() { + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + MigrationOperationSnapshot copying = copying(53); + store.create(pending()); + store.compareAndTransition(OPERATION_ID, MigrationOperationState.PENDING, copying); + DurableCutoverFailureFinalizer finalizer = new DurableCutoverFailureFinalizer(store); + + assertStoreError(SetupErrorCode.OPERATION_CONFLICT, () -> finalizer.finalizeFailure( + OPERATION_ID, DurableKnownFailure.VERIFICATION, COMPLETED)); + assertThatThrownBy(() -> finalizer.finalizeFailure( + OPERATION_ID, DurableKnownFailure.COPY, STARTED.minusNanos(1))) + .isInstanceOf(IllegalArgumentException.class) + .hasNoCause(); + assertThat(store.find(OPERATION_ID)).contains(copying); + } + + @Test + void exactTerminalReplayReturnsAlreadyConfirmedWithoutChangingCompletionIdentity() { + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + store.create(pending()); + store.compareAndTransition(OPERATION_ID, MigrationOperationState.PENDING, copying(61)); + DurableCutoverFailureFinalizer finalizer = new DurableCutoverFailureFinalizer(store); + assertThat(finalizer.finalizeFailure( + OPERATION_ID, DurableKnownFailure.COPY, COMPLETED)) + .isEqualTo(Disposition.TRANSITIONED); + + assertThat(finalizer.finalizeFailure( + OPERATION_ID, DurableKnownFailure.COPY, COMPLETED)) + .isEqualTo(Disposition.ALREADY_CONFIRMED); + assertStoreError(SetupErrorCode.OPERATION_CONFLICT, () -> finalizer.finalizeFailure( + OPERATION_ID, DurableKnownFailure.COPY, COMPLETED.plusSeconds(1))); + } + + @Test + void preStartTerminalFailureAlwaysConflictsInsteadOfApplyingRunningTimeValidation() { + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + store.create(pending()); + MigrationOperationSnapshot unsupported = new MigrationOperationSnapshot( + OPERATION_ID, MigrationOperationState.FAILED, MigrationTarget.MYSQL, + ApplyMode.MANAGED_WRITE, MigrationStage.FAILED, 0, CREATED, null, CREATED.plusSeconds(2), + VerificationState.PENDING, SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED, null, 0, + false, false, false, IDENTITY, GENERATION); + store.compareAndTransition(OPERATION_ID, MigrationOperationState.PENDING, unsupported); + + assertStoreError(SetupErrorCode.OPERATION_CONFLICT, () -> + new DurableCutoverFailureFinalizer(store).finalizeFailure( + OPERATION_ID, DurableKnownFailure.COPY, COMPLETED)); + } + + @Test + void committedFailureAndUncertainConfirmationRequireExactReplay() { + FileMigrationOperationStore initial = new FileMigrationOperationStore(root); + initial.create(pending()); + initial.compareAndTransition(OPERATION_ID, MigrationOperationState.PENDING, copying(62)); + MigrationOperationFilePublisher committed = new MigrationOperationFilePublisher(root); + AtomicInteger publications = new AtomicInteger(); + FileMigrationOperationStore uncertain = new FileMigrationOperationStore(root, (target, content) -> { + committed.publish(target, content); + if (publications.incrementAndGet() <= 2) { + throw new CommittedSetupFileDurabilityException(); + } + }); + DurableCutoverFailureFinalizer finalizer = new DurableCutoverFailureFinalizer(uncertain); + + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, () -> finalizer.finalizeFailure( + OPERATION_ID, DurableKnownFailure.COPY, COMPLETED)); + assertThat(initial.find(OPERATION_ID).orElseThrow().errorCode()) + .isEqualTo(SetupErrorCode.MIGRATION_COPY_FAILED); + + assertThat(finalizer.finalizeFailure( + OPERATION_ID, DurableKnownFailure.COPY, COMPLETED)) + .isEqualTo(Disposition.ALREADY_CONFIRMED); + assertThat(publications).hasValue(3); + } + + @Test + @Timeout(10) + void progressTransitionAndFailureFinalizationSerializeUnderTheStoreLock() throws Exception { + FileMigrationOperationStore initial = new FileMigrationOperationStore(root); + initial.create(pending()); + initial.compareAndTransition(OPERATION_ID, MigrationOperationState.PENDING, copying(12)); + MigrationOperationFilePublisher committed = new MigrationOperationFilePublisher(root); + CountDownLatch progressPublished = new CountDownLatch(1); + CountDownLatch releaseProgress = new CountDownLatch(1); + FileMigrationOperationStore updating = new FileMigrationOperationStore(root, (target, content) -> { + committed.publish(target, content); + progressPublished.countDown(); + try { + releaseProgress.await(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new CommittedSetupFileDurabilityException(); + } + }); + DurableCutoverFailureFinalizer finalizer = new DurableCutoverFailureFinalizer( + new FileMigrationOperationStore(root)); + CountDownLatch finalizationStarted = new CountDownLatch(1); + try (ExecutorService executor = Executors.newFixedThreadPool(2)) { + Future progress = executor.submit(() -> updating.compareAndTransition( + OPERATION_ID, MigrationOperationState.RUNNING, copying(63))); + assertThat(progressPublished.await(5, TimeUnit.SECONDS)).isTrue(); + Future failure = executor.submit(() -> { + finalizationStarted.countDown(); + return finalizer.finalizeFailure( + OPERATION_ID, DurableKnownFailure.COPY, COMPLETED); + }); + assertThat(finalizationStarted.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(failure.isDone()).isFalse(); + releaseProgress.countDown(); + progress.get(5, TimeUnit.SECONDS); + assertThat(failure.get(5, TimeUnit.SECONDS)).isEqualTo(Disposition.TRANSITIONED); + } finally { + releaseProgress.countDown(); + } + + assertThat(initial.find(OPERATION_ID).orElseThrow().progressPercent()).isEqualTo(63); + } + + @Test + void surfaceAndFailuresContainNoPayloadOrThrowableChannel() { + assertThat(DurableCutoverFailureFinalizer.class.getDeclaredMethods()) + .allSatisfy(method -> assertThat(method.getParameterTypes()) + .doesNotContain(Throwable.class, Exception.class, Error.class)); + assertThat(new DurableCutoverFailureFinalizer(new FileMigrationOperationStore(root)).toString()) + .doesNotContain("jdbc:", "password", "username", IDENTITY, GENERATION); + } + + private static MigrationOperationSnapshot pending() { + return new MigrationOperationSnapshot( + OPERATION_ID, MigrationOperationState.PENDING, MigrationTarget.MYSQL, + ApplyMode.MANAGED_WRITE, MigrationStage.QUEUED, 0, CREATED, null, null, + VerificationState.PENDING, null, null, 1000, false, false, false, + IDENTITY, GENERATION); + } + + private static MigrationOperationSnapshot copying(int progress) { + return new MigrationOperationSnapshot( + OPERATION_ID, MigrationOperationState.RUNNING, MigrationTarget.MYSQL, + ApplyMode.MANAGED_WRITE, MigrationStage.COPYING, progress, CREATED, STARTED, null, + VerificationState.PENDING, null, null, 1000, false, false, false, + IDENTITY, GENERATION); + } + + private static MigrationOperationSnapshot verifying() { + return new MigrationOperationSnapshot( + OPERATION_ID, MigrationOperationState.RUNNING, MigrationTarget.MYSQL, + ApplyMode.MANAGED_WRITE, MigrationStage.VERIFYING, 100, CREATED, STARTED, null, + VerificationState.RUNNING, null, null, 1000, false, false, false, + IDENTITY, GENERATION); + } + + private static void assertStoreError(SetupErrorCode code, ThrowingAction action) { + assertThatThrownBy(action::run) + .isInstanceOfSatisfying(MigrationOperationStoreException.class, + failure -> assertThat(failure.errorCode()).isEqualTo(code)) + .hasNoCause(); + } + + @FunctionalInterface + private interface ThrowingAction { + void run(); + } +} From 54be0dfde09564c2ad06af6ffb68d74ac9803222 Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 20:58:38 +0800 Subject: [PATCH 66/71] Coordinate managed migration commands --- .../DeploymentMigrationCommandRunner.java | 259 ++++++++++ .../DurableCutoverFailureFinalizer.java | 9 +- .../setup/workflow/DurableKnownFailure.java | 26 +- .../workflow/FileMigrationOperationStore.java | 22 + .../MigrationCandidateGeneration.java | 33 ++ .../workflow/MigrationCommandCompletion.java | 85 +++ .../setup/workflow/MigrationCommandDraft.java | 37 ++ .../workflow/MigrationCommandSubmission.java | 64 +++ .../setup/workflow/MigrationCommandTask.java | 189 +++++++ .../workflow/MigrationCommandTaskFactory.java | 100 ++++ .../workflow/MigrationCommandWorker.java | 28 + .../MigrationFailureFinalization.java | 64 +++ .../MigrationOperationProjection.java | 27 + .../workflow/MigrationPreparationBarrier.java | 176 +++++++ .../workflow/MigrationProgressJournal.java | 74 +++ .../workflow/MigrationTargetRequest.java | 28 + .../workflow/RetainedCutoverCoordinator.java | 5 + .../RetainedCutoverRecoveryPhase.java | 16 + .../setup/workflow/RetainedCutoverState.java | 14 + ...mentMigrationCommandRunnerFailureTest.java | 312 +++++++++++ ...entMigrationCommandRunnerRecoveryTest.java | 487 ++++++++++++++++++ ...ymentMigrationCommandRunnerReviewTest.java | 444 ++++++++++++++++ .../DeploymentMigrationCommandRunnerTest.java | 451 ++++++++++++++++ .../DurableCutoverFailureFinalizerTest.java | 5 +- .../FileMigrationOperationStoreExactTest.java | 21 + .../MigrationPreparationBarrierTest.java | 66 +++ 26 files changed, 3036 insertions(+), 6 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationCommandRunner.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCandidateGeneration.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCommandCompletion.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCommandDraft.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCommandSubmission.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCommandTask.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCommandTaskFactory.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCommandWorker.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationFailureFinalization.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationProjection.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationPreparationBarrier.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationProgressJournal.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationTargetRequest.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverRecoveryPhase.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationCommandRunnerFailureTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationCommandRunnerRecoveryTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationCommandRunnerReviewTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationCommandRunnerTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationPreparationBarrierTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationCommandRunner.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationCommandRunner.java new file mode 100644 index 0000000000..376a6c2f83 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationCommandRunner.java @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.time.Clock; +import java.time.Duration; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MetadataMigrationRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationView; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction; + +/** Single-slot, Spring-free command boundary for managed metadata migration. */ +final class DeploymentMigrationCommandRunner implements AutoCloseable { + + private final FileMigrationOperationStore store; + private final MigrationCommandTaskFactory taskFactory; + private final Duration timeout; + private final ExecutorService worker; + private MigrationCommandTask active; + private boolean closed; + + DeploymentMigrationCommandRunner( + FileMigrationOperationStore store, + ManagedMigrationConfigurationTransaction configuration, + RetainedCutoverCoordinator coordinator, + Clock clock, + Duration timeout) { + this(store, configuration, coordinator, clock, timeout, MigrationCommandWorker.create()); + } + + /** Test seam for deterministic worker scheduling and rejection. */ + DeploymentMigrationCommandRunner( + FileMigrationOperationStore store, + ManagedMigrationConfigurationTransaction configuration, + RetainedCutoverCoordinator coordinator, + Clock clock, + Duration timeout, + ExecutorService worker) { + this.store = Objects.requireNonNull(store, "store"); + this.timeout = requirePositive(timeout); + taskFactory = new MigrationCommandTaskFactory( + store, configuration, coordinator, clock, this.timeout); + this.worker = Objects.requireNonNull(worker, "worker"); + } + + MigrationView start(MetadataMigrationRequest request) { + Objects.requireNonNull(request, "request"); + requireManaged(request.applyMode()); + MigrationTargetRequest target = taskFactory.request(request); + MigrationPreparationBarrier barrier; + synchronized (this) { + requireOpen(); + if (active != null) { + if (!active.operationId().equals(target.operationId()) || !active.matches(target)) { + throw failure(SetupErrorCode.OPERATION_CONFLICT); + } + barrier = active.barrier(); + RetainedCutoverRecoveryPhase recovery = active.claimRetry(); + if (recovery != null) { + submitRetry(active, recovery); + } + } else { + Optional persisted = store.selectForStartup(target.operationId()); + if (persisted.isPresent() && persisted.get().state() != MigrationOperationState.PENDING) { + MigrationCommandTaskFactory.requireCompatible(persisted.get(), target); + return MigrationOperationProjection.view( + store.confirmExactForStartup(persisted.get())); + } + active = taskFactory.create( + this, target, persisted.orElse(null), request.targetDatabase().password()); + barrier = active.barrier(); + submit(active); + } + } + return barrier.await(timeout); + } + + Optional find(String operationId) { + Optional persisted = store.find(operationId); + synchronized (this) { + if (active != null && active.operationId().equals(operationId) + && projectionRequiresSettlement(active, persisted)) { + throw failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + } + return persisted.map(MigrationOperationProjection::view); + } + + synchronized Optional activeOperationId() { + if (active != null) { + return Optional.of(active.operationId()); + } + return store.selectUniqueNonterminalForStartup() + .map(MigrationOperationSnapshot::operationId); + } + + synchronized Optional activeRecoveryPhase() { + if (active == null || active.executing()) { + return Optional.empty(); + } + return Optional.of(active.recoveryPhase()); + } + + synchronized void finished( + MigrationCommandTask task, RetainedCutoverRecoveryPhase recoveryPhase) { + task.completed(recoveryPhase); + if (active == task && recoveryPhase == RetainedCutoverRecoveryPhase.NONE) { + active = null; + } + notifyAll(); + } + + @Override + public void close() { + boolean interrupted = false; + Throwable failure = null; + while (failure == null) { + MigrationCommandTask retryTask; + RetainedCutoverRecoveryPhase recovery; + synchronized (this) { + closed = true; + while (active != null && active.executing()) { + try { + wait(); + } catch (InterruptedException waitInterrupted) { + interrupted = true; + } + } + if (active == null) { + break; + } + retryTask = active; + recovery = retryTask.claimRetry(); + } + try { + retryTask.retry(recovery); + } catch (RuntimeException | Error retryFailure) { + failure = retryFailure; + } + } + synchronized (this) { + if (active == null) { + worker.shutdown(); + } + } + if (activeOperationInMemory().isEmpty()) { + while (!worker.isTerminated()) { + try { + worker.awaitTermination(1, TimeUnit.DAYS); + } catch (InterruptedException waitInterrupted) { + interrupted = true; + } + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + rethrow(failure); + } + + private void submit(MigrationCommandTask task) { + try { + worker.execute(task); + } catch (Error fatal) { + reject(task); + throw fatal; + } catch (RejectedExecutionException rejected) { + reject(task); + throw failure(SetupErrorCode.MIGRATION_UNAVAILABLE); + } catch (RuntimeException unexpected) { + reject(task); + throw failure(SetupErrorCode.MIGRATION_UNAVAILABLE); + } + } + + private void submitRetry( + MigrationCommandTask task, RetainedCutoverRecoveryPhase recovery) { + try { + MigrationCommandSubmission.submitWhenAvailable( + worker, () -> task.retry(recovery), timeout); + } catch (Error fatal) { + task.restoreRetry(recovery); + throw fatal; + } catch (MetadataMigrationException knownFailure) { + task.restoreRetry(recovery); + throw knownFailure; + } catch (RejectedExecutionException rejected) { + task.restoreRetry(recovery); + throw failure(SetupErrorCode.MIGRATION_UNAVAILABLE); + } catch (RuntimeException unexpected) { + task.restoreRetry(recovery); + throw failure(SetupErrorCode.MIGRATION_UNAVAILABLE); + } + } + + private void reject(MigrationCommandTask task) { + task.reject(); + if (active == task) { + active = null; + notifyAll(); + } + } + + private void requireOpen() { + if (closed) { + throw failure(SetupErrorCode.MIGRATION_UNAVAILABLE); + } + } + + private static void requireManaged(ApplyMode applyMode) { + if (applyMode != ApplyMode.MANAGED_WRITE) { + throw failure(SetupErrorCode.INVALID_REQUEST); + } + } + + private static Duration requirePositive(Duration value) { + if (value == null || value.isZero() || value.isNegative()) { + throw new IllegalArgumentException("Migration timeout must be positive"); + } + return value; + } + + private static boolean projectionRequiresSettlement( + MigrationCommandTask task, Optional persisted) { + boolean actionableOrMissing = persisted.isEmpty() || persisted.filter(snapshot -> + snapshot.terminal() || snapshot.state() != MigrationOperationState.PENDING + && snapshot.state() != MigrationOperationState.RUNNING).isPresent(); + return task.projectionRequiresSettlement(actionableOrMissing); + } + + private synchronized Optional activeOperationInMemory() { + return Optional.ofNullable(active).map(MigrationCommandTask::operationId); + } + + private static void rethrow(Throwable failure) { + if (failure instanceof Error fatal) { + throw fatal; + } + if (failure instanceof RuntimeException runtime) { + throw runtime; + } + } + + private static MigrationOperationStoreException failure(SetupErrorCode code) { + return new MigrationOperationStoreException(code); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverFailureFinalizer.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverFailureFinalizer.java index aade95e8db..c9cb23e71a 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverFailureFinalizer.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverFailureFinalizer.java @@ -35,14 +35,15 @@ final class DurableCutoverFailureFinalizer { private MigrationOperationSnapshot replacement( MigrationOperationSnapshot current, DurableKnownFailure failure, Instant completedAt) { + DurableKnownFailure resolved = failure.resolve(current); if (current.state() == MigrationOperationState.FAILED) { - if (current.errorCode() == failure.errorCode() && current.completedAt().equals(completedAt)) { + if (current.errorCode() == resolved.errorCode() && current.completedAt().equals(completedAt)) { return current; } throw conflict(); } if (current.state() != MigrationOperationState.RUNNING - || current.stage() != failure.requiredStage()) { + || current.stage() != resolved.requiredStage()) { throw conflict(); } if (completedAt.isBefore(current.startedAt())) { @@ -50,8 +51,8 @@ final class DurableCutoverFailureFinalizer { } return new MigrationOperationSnapshot( current.operationId(), MigrationOperationState.FAILED, current.target(), current.applyMode(), - MigrationStage.FAILED, failure.progress(current), current.createdAt(), current.startedAt(), - completedAt, failure.verificationState(), failure.errorCode(), null, 0, + MigrationStage.FAILED, resolved.progress(current), current.createdAt(), current.startedAt(), + completedAt, resolved.verificationState(), resolved.errorCode(), null, 0, false, false, false, current.targetIdentityHash(), current.managedCandidateGeneration()); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableKnownFailure.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableKnownFailure.java index a4066cea9b..da48986559 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableKnownFailure.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableKnownFailure.java @@ -7,6 +7,7 @@ package org.apache.hertzbeat.manager.setup.workflow; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; @@ -17,7 +18,8 @@ enum DurableKnownFailure { VERIFICATION( SetupErrorCode.MIGRATION_VERIFICATION_FAILED, MigrationStage.VERIFYING, - VerificationState.FAILED); + VerificationState.FAILED), + CURRENT_PHASE(null, null, null); private final SetupErrorCode errorCode; private final MigrationStage requiredStage; @@ -47,4 +49,26 @@ enum DurableKnownFailure { int progress(MigrationOperationSnapshot current) { return this == COPY ? current.progressPercent() : 100; } + + DurableKnownFailure resolve(MigrationOperationSnapshot current) { + if (this != CURRENT_PHASE) { + return this; + } + if (current.state() == MigrationOperationState.FAILED) { + return switch (current.errorCode()) { + case MIGRATION_COPY_FAILED -> COPY; + case MIGRATION_VERIFICATION_FAILED -> VERIFICATION; + default -> throw conflict(); + }; + } + return switch (current.stage()) { + case COPYING -> COPY; + case VERIFYING -> VERIFICATION; + default -> throw conflict(); + }; + } + + private static MigrationOperationStoreException conflict() { + return new MigrationOperationStoreException(SetupErrorCode.OPERATION_CONFLICT); + } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java index 8fb01f4a08..492d486faf 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java @@ -115,6 +115,28 @@ public final class FileMigrationOperationStore implements MigrationOperationStor }); } + /** Confirms one fully equal startup snapshot and its containing collection durably. */ + MigrationOperationSnapshot confirmExactForStartup(MigrationOperationSnapshot expected) { + Objects.requireNonNull(expected, "expected"); + requireSafeId(expected.operationId()); + return locked(() -> { + List snapshots = read(); + if (snapshots.stream().anyMatch(snapshot -> !snapshot.terminal() + && !snapshot.operationId().equals(expected.operationId()))) { + throw failure(SetupErrorCode.OPERATION_CONFLICT); + } + MigrationOperationSnapshot current = snapshots.stream() + .filter(snapshot -> snapshot.operationId().equals(expected.operationId())) + .findFirst() + .orElseThrow(() -> failure(SetupErrorCode.OPERATION_NOT_FOUND)); + if (!current.equals(expected)) { + throw failure(SetupErrorCode.OPERATION_CONFLICT); + } + writeAndConfirm(snapshots); + return current; + }); + } + /** Selects the only nonterminal startup record under the operation-store lock. */ Optional selectUniqueNonterminalForStartup() { return locked(() -> { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCandidateGeneration.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCandidateGeneration.java new file mode 100644 index 0000000000..146fce06d5 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCandidateGeneration.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; +import org.apache.hertzbeat.manager.setup.api.OperationIdValidator; + +/** Deterministic, secret-free candidate generation derived from a validated operation id. */ +final class MigrationCandidateGeneration { + + private MigrationCandidateGeneration() { + } + + static String fromOperationId(String operationId) { + if (!OperationIdValidator.isSafe(operationId)) { + throw new IllegalArgumentException("Invalid migration operation id"); + } + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(operationId.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException impossible) { + throw new IllegalStateException("SHA-256 is unavailable"); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCommandCompletion.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCommandCompletion.java new file mode 100644 index 0000000000..ba66776579 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCommandCompletion.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Resolves retained ownership without allowing a recovery probe to skip task completion. */ +final class MigrationCommandCompletion { + + private MigrationCommandCompletion() { + } + + static Throwable finish( + RetainedCutoverCoordinator coordinator, + String operationId, + boolean finalizationPending, + Throwable primary, + CompletionSink sink) { + RetainedCutoverRecoveryPhase phase = localPhase(finalizationPending, primary); + Throwable effective = primary; + try { + if (!finalizationPending) { + RetainedCutoverRecoveryPhase exact = coordinator.recoveryPhase(operationId); + if (exact != null) { + phase = exact; + } + } + } catch (RuntimeException | Error probeFailure) { + effective = prioritize(primary, probeFailure); + } finally { + sink.completed(phase); + } + return effective; + } + + private static RetainedCutoverRecoveryPhase localPhase( + boolean finalizationPending, Throwable failure) { + if (finalizationPending) { + return RetainedCutoverRecoveryPhase.FAILURE_FINALIZATION_PENDING; + } + if (failure instanceof RetainedCopyJournalHandoffException) { + return RetainedCutoverRecoveryPhase.HANDOFF_PENDING; + } + if (failure instanceof RetainedCutoverReleaseRequiredException + || failure instanceof Error fatal && hasReleaseMarker(fatal)) { + return RetainedCutoverRecoveryPhase.RELEASE_PENDING; + } + return RetainedCutoverRecoveryPhase.NONE; + } + + private static Throwable prioritize(Throwable primary, Throwable secondary) { + if (primary instanceof Error fatal) { + fatal.addSuppressed(recoveryMarker()); + return fatal; + } + if (secondary instanceof Error fatal) { + fatal.addSuppressed(recoveryMarker()); + return fatal; + } + return primary == null ? recoveryMarker() : primary; + } + + private static boolean hasReleaseMarker(Error fatal) { + for (Throwable suppressed : fatal.getSuppressed()) { + if (suppressed instanceof RetainedCutoverReleaseRequiredException) { + return true; + } + } + return false; + } + + private static MigrationOperationStoreException recoveryMarker() { + return new MigrationOperationStoreException(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + + @FunctionalInterface + interface CompletionSink { + void completed(RetainedCutoverRecoveryPhase phase); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCommandDraft.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCommandDraft.java new file mode 100644 index 0000000000..40722b103c --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCommandDraft.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.time.Instant; +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; + +/** Immutable command identity whose start time is bound only by the executing worker. */ +record MigrationCommandDraft( + String operationId, + MigrationTarget target, + ApplyMode applyMode, + Instant createdAt, + String candidateGeneration) { + + MigrationCommandDraft { + Objects.requireNonNull(operationId, "operationId"); + Objects.requireNonNull(target, "target"); + Objects.requireNonNull(applyMode, "applyMode"); + Objects.requireNonNull(createdAt, "createdAt"); + Objects.requireNonNull(candidateGeneration, "candidateGeneration"); + } + + DurableCutoverDraft start(Instant workerStartedAt) { + Objects.requireNonNull(workerStartedAt, "workerStartedAt"); + Instant safeStartedAt = workerStartedAt.isBefore(createdAt) ? createdAt : workerStartedAt; + return new DurableCutoverDraft( + operationId, target, applyMode, createdAt, safeStartedAt, candidateGeneration); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCommandSubmission.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCommandSubmission.java new file mode 100644 index 0000000000..52e40fae35 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCommandSubmission.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.time.Duration; +import java.util.Objects; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.LockSupport; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Bounded handoff to a zero-queue worker that may still be unwinding its previous task. */ +final class MigrationCommandSubmission { + + private static final long AVAILABILITY_POLL_NANOS = TimeUnit.MILLISECONDS.toNanos(1); + + private MigrationCommandSubmission() { + } + + static void submitWhenAvailable( + ExecutorService worker, Runnable task, Duration timeout) { + Objects.requireNonNull(worker, "worker"); + Objects.requireNonNull(task, "task"); + JdbcMetadataMigrationDeadline deadline = JdbcMetadataMigrationDeadline.start( + timeout, System::nanoTime); + boolean interrupted = false; + try { + while (deadline.remainingNanos() > 0) { + try { + worker.execute(task); + return; + } catch (RejectedExecutionException transientRejection) { + if (worker.isShutdown()) { + throw unavailable(); + } + long remaining = deadline.remainingNanos(); + if (remaining <= 0) { + throw unavailable(); + } + LockSupport.parkNanos(Math.min(remaining, AVAILABILITY_POLL_NANOS)); + if (Thread.interrupted()) { + interrupted = true; + throw new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT); + } + } + } + throw unavailable(); + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + + private static MigrationOperationStoreException unavailable() { + return new MigrationOperationStoreException(SetupErrorCode.MIGRATION_UNAVAILABLE); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCommandTask.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCommandTask.java new file mode 100644 index 0000000000..3be6199e35 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCommandTask.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.time.Clock; +import java.time.Duration; +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; + +/** Owns one command's copied password until the retained coordinator has fully returned. */ +final class MigrationCommandTask implements Runnable { + + private final DeploymentMigrationCommandRunner owner; + private final RetainedCutoverCoordinator coordinator; + private final FileMigrationOperationStore store; + private final ManagedMigrationConfigurationTransaction configuration; + private final MigrationCommandDraft draft; + private final MetadataDatabaseSettings target; + private final SecretValue password; + private final Duration timeout; + private final Clock clock; + private final MigrationPreparationBarrier barrier; + private final MigrationFailureFinalization failureFinalization; + private RetainedCutoverRecoveryPhase recoveryPhase = RetainedCutoverRecoveryPhase.NONE; + private boolean executing = true; + + MigrationCommandTask( + DeploymentMigrationCommandRunner owner, + RetainedCutoverCoordinator coordinator, + FileMigrationOperationStore store, + ManagedMigrationConfigurationTransaction configuration, + MigrationCommandDraft draft, + MetadataDatabaseSettings target, + SecretValue password, + Duration timeout, + Clock clock, + MigrationPreparationBarrier barrier) { + this.owner = Objects.requireNonNull(owner, "owner"); + this.coordinator = Objects.requireNonNull(coordinator, "coordinator"); + this.store = Objects.requireNonNull(store, "store"); + this.configuration = Objects.requireNonNull(configuration, "configuration"); + this.draft = Objects.requireNonNull(draft, "draft"); + this.target = Objects.requireNonNull(target, "target"); + this.password = Objects.requireNonNull(password, "password"); + this.timeout = Objects.requireNonNull(timeout, "timeout"); + this.clock = Objects.requireNonNull(clock, "clock"); + this.barrier = Objects.requireNonNull(barrier, "barrier"); + failureFinalization = new MigrationFailureFinalization(draft.operationId(), store, clock); + } + + @Override + public void run() { + Throwable failure = null; + try { + DurableCutoverDraft runningDraft = draft.start(clock.instant()); + barrier.bind( + runningDraft, new DurableCutoverPreparation(runningDraft, store, configuration)); + coordinator.execute( + draft.operationId(), target, password, timeout, + new MigrationProgressJournal(draft.operationId(), store), barrier, + new DurableRetainedCopyJournalHandoff(runningDraft, store)); + } catch (MetadataMigrationException knownFailure) { + barrier.workerFailed(knownFailure); + failure = knownFailure; + try { + failureFinalization.finalizeKnown(knownFailure); + } catch (RuntimeException finalizationFailure) { + failure = finalizationFailure; + } catch (Error finalizationFatal) { + failure = finalizationFatal; + } + } catch (Error fatal) { + barrier.workerFailed(fatal); + failure = fatal; + } catch (RuntimeException runtimeFailure) { + barrier.workerFailed(runtimeFailure); + failure = runtimeFailure; + } finally { + password.close(); + failure = finish(failure); + } + rethrowFatal(failure); + } + + void retry(RetainedCutoverRecoveryPhase expectedPhase) { + Throwable failure = null; + try { + if (expectedPhase == RetainedCutoverRecoveryPhase.FAILURE_FINALIZATION_PENDING) { + failureFinalization.retry(); + } else if (expectedPhase == RetainedCutoverRecoveryPhase.RELEASE_PENDING) { + coordinator.retryRelease(draft.operationId(), timeout); + } else if (expectedPhase == RetainedCutoverRecoveryPhase.HANDOFF_PENDING) { + coordinator.retryHandoff(draft.operationId()); + } else { + throw new MigrationOperationStoreException( + SetupErrorCode.OPERATION_CONFLICT); + } + } catch (MetadataMigrationException knownFailure) { + failureFinalization.finalizeKnown(knownFailure); + failure = knownFailure; + } catch (RuntimeException | Error retryFailure) { + failure = retryFailure; + } finally { + failure = finish(failure); + } + if (recoveryPhase() != RetainedCutoverRecoveryPhase.NONE || failure instanceof Error) { + rethrow(failure); + } + } + + String operationId() { + return draft.operationId(); + } + + boolean matches(MigrationTargetRequest request) { + return draft.target() == request.target() + && draft.applyMode() == request.applyMode() + && target.equals(request.settings()); + } + + MigrationPreparationBarrier barrier() { + return barrier; + } + + synchronized RetainedCutoverRecoveryPhase claimRetry() { + if (executing || recoveryPhase == RetainedCutoverRecoveryPhase.NONE) { + return null; + } + executing = true; + return recoveryPhase; + } + + synchronized void restoreRetry(RetainedCutoverRecoveryPhase phase) { + executing = false; + recoveryPhase = Objects.requireNonNull(phase, "phase"); + } + + synchronized boolean executing() { + return executing; + } + + synchronized RetainedCutoverRecoveryPhase recoveryPhase() { + return recoveryPhase; + } + + synchronized boolean projectionRequiresSettlement(boolean actionableOrMissing) { + return recoveryPhase != RetainedCutoverRecoveryPhase.NONE + || failureFinalization.pending() + || executing && actionableOrMissing; + } + + synchronized void completed(RetainedCutoverRecoveryPhase phase) { + recoveryPhase = Objects.requireNonNull(phase, "phase"); + executing = false; + } + + void reject() { + password.close(); + } + + private Throwable finish(Throwable primary) { + return MigrationCommandCompletion.finish( + coordinator, draft.operationId(), failureFinalization.pending(), primary, + phase -> owner.finished(this, phase)); + } + + private static void rethrowFatal(Throwable failure) { + if (failure instanceof Error fatal) { + throw fatal; + } + } + + private static void rethrow(Throwable failure) { + if (failure instanceof Error fatal) { + throw fatal; + } + if (failure instanceof RuntimeException runtime) { + throw runtime; + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCommandTaskFactory.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCommandTaskFactory.java new file mode 100644 index 0000000000..6f6c560d33 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCommandTaskFactory.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MetadataMigrationRequest; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; + +/** Builds one command task while keeping request and credential material out of retained state. */ +final class MigrationCommandTaskFactory { + + private final FileMigrationOperationStore store; + private final ManagedMigrationConfigurationTransaction configuration; + private final RetainedCutoverCoordinator coordinator; + private final Clock clock; + private final Duration timeout; + + MigrationCommandTaskFactory( + FileMigrationOperationStore store, + ManagedMigrationConfigurationTransaction configuration, + RetainedCutoverCoordinator coordinator, + Clock clock, + Duration timeout) { + this.store = Objects.requireNonNull(store, "store"); + this.configuration = Objects.requireNonNull(configuration, "configuration"); + this.coordinator = Objects.requireNonNull(coordinator, "coordinator"); + this.clock = Objects.requireNonNull(clock, "clock"); + this.timeout = Objects.requireNonNull(timeout, "timeout"); + } + + MigrationTargetRequest request(MetadataMigrationRequest request) { + MetadataDatabaseConfiguration database = request.targetDatabase(); + return new MigrationTargetRequest( + request.operationId(), request.target(), request.applyMode(), + new MetadataDatabaseSettings(database.kind(), database.jdbcUrl(), database.username())); + } + + MigrationCommandTask create( + DeploymentMigrationCommandRunner owner, + MigrationTargetRequest target, + MigrationOperationSnapshot pending, + String password) { + MigrationCommandDraft draft = draft(target, pending); + SecretValue ownedPassword = null; + try { + try (SecretValue borrowed = SecretValue.of(password)) { + ownedPassword = SecretValue.copyOf(borrowed); + } + MigrationPreparationBarrier barrier = new MigrationPreparationBarrier(store); + return new MigrationCommandTask( + owner, coordinator, store, configuration, draft, target.settings(), ownedPassword, + timeout, clock, barrier); + } catch (Error fatal) { + close(ownedPassword); + throw fatal; + } catch (RuntimeException failure) { + close(ownedPassword); + throw failure; + } + } + + static void requireCompatible( + MigrationOperationSnapshot snapshot, MigrationTargetRequest request) { + if (snapshot.target() != request.target() || snapshot.applyMode() != request.applyMode()) { + throw new MigrationOperationStoreException(SetupErrorCode.OPERATION_CONFLICT); + } + } + + private MigrationCommandDraft draft( + MigrationTargetRequest request, MigrationOperationSnapshot pending) { + String generation = MigrationCandidateGeneration.fromOperationId(request.operationId()); + Instant createdAt = pending == null ? clock.instant() : pending.createdAt(); + if (pending != null) { + requireCompatible(pending, request); + if (!generation.equals(pending.managedCandidateGeneration())) { + throw new MigrationOperationStoreException(SetupErrorCode.OPERATION_CONFLICT); + } + } + return new MigrationCommandDraft( + request.operationId(), request.target(), request.applyMode(), createdAt, generation); + } + + private static void close(SecretValue secret) { + if (secret != null) { + secret.close(); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCommandWorker.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCommandWorker.java new file mode 100644 index 0000000000..a811cee17f --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationCommandWorker.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** Creates the bounded one-operation migration command worker. */ +final class MigrationCommandWorker { + + private MigrationCommandWorker() { + } + + static ThreadPoolExecutor create() { + return new ThreadPoolExecutor( + 1, 1, 0, TimeUnit.MILLISECONDS, new SynchronousQueue<>(), task -> { + Thread thread = new Thread(task, "metadata-migration-command"); + thread.setDaemon(true); + return thread; + }, new ThreadPoolExecutor.AbortPolicy()); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationFailureFinalization.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationFailureFinalization.java new file mode 100644 index 0000000000..8b65bebc41 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationFailureFinalization.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.time.Clock; +import java.time.Instant; +import java.util.Objects; + +/** Pins and retries one exact durable known-cleanup failure replacement. */ +final class MigrationFailureFinalization { + + private final String operationId; + private final DurableCutoverFailureFinalizer finalizer; + private final Clock clock; + private DurableKnownFailure pendingFailure; + private Instant completedAt; + + MigrationFailureFinalization( + String operationId, FileMigrationOperationStore store, Clock clock) { + this.operationId = Objects.requireNonNull(operationId, "operationId"); + finalizer = new DurableCutoverFailureFinalizer(store); + this.clock = Objects.requireNonNull(clock, "clock"); + } + + void finalizeKnown(MetadataMigrationException failure) { + DurableKnownFailure known = switch (failure.code()) { + case SCHEMA, COPY, VERIFICATION, SEQUENCE -> DurableKnownFailure.CURRENT_PHASE; + default -> null; + }; + if (known == null) { + return; + } + synchronized (this) { + if (pendingFailure == null) { + pendingFailure = known; + completedAt = clock.instant(); + } + } + retry(); + } + + void retry() { + DurableKnownFailure known; + Instant pinnedCompletedAt; + synchronized (this) { + known = Objects.requireNonNull(pendingFailure, "pendingFailure"); + pinnedCompletedAt = Objects.requireNonNull(completedAt, "completedAt"); + } + finalizer.finalizeFailure(operationId, known, pinnedCompletedAt); + synchronized (this) { + pendingFailure = null; + completedAt = null; + } + } + + synchronized boolean pending() { + return pendingFailure != null; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationProjection.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationProjection.java new file mode 100644 index 0000000000..a05b4e9a5b --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationOperationProjection.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationView; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +/** Maps durable internal state to the frozen secret-free API projection. */ +final class MigrationOperationProjection { + + private MigrationOperationProjection() { + } + + static MigrationView view(MigrationOperationSnapshot snapshot) { + return new MigrationView( + snapshot.operationId(), snapshot.state(), MetadataDatabaseKind.H2, snapshot.target(), + snapshot.stage(), snapshot.progressPercent(), snapshot.createdAt(), snapshot.startedAt(), + snapshot.completedAt(), snapshot.verificationState(), snapshot.errorCode(), + snapshot.nextPollAfterMillis(), snapshot.activationAvailable(), snapshot.restartRequired(), + snapshot.externalApplyRequired()); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationPreparationBarrier.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationPreparationBarrier.java new file mode 100644 index 0000000000..dfb4676d29 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationPreparationBarrier.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.time.Duration; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationView; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; + +/** Publishes an authoritative journal view at the durable preparation boundary. */ +final class MigrationPreparationBarrier implements RetainedCutoverPreparation { + + private final FileMigrationOperationStore store; + private final CompletableFuture prepared = new CompletableFuture<>(); + private DurableCutoverDraft draft; + private RetainedCutoverPreparation delegate; + + MigrationPreparationBarrier(FileMigrationOperationStore store) { + this.store = Objects.requireNonNull(store, "store"); + } + + synchronized void bind( + DurableCutoverDraft boundDraft, RetainedCutoverPreparation boundDelegate) { + if (draft != null || delegate != null) { + throw new MigrationOperationStoreException(SetupErrorCode.OPERATION_CONFLICT); + } + draft = Objects.requireNonNull(boundDraft, "draft"); + delegate = Objects.requireNonNull(boundDelegate, "delegate"); + } + + @Override + public void prepare( + RetainedCutoverPreparationContext context, + MetadataDatabaseSettings target, + SecretValue borrowedPassword) { + try { + requireDelegate().prepare(context, target, borrowedPassword); + } catch (Error fatal) { + publishFatal(context, fatal); + throw fatal; + } catch (RuntimeException failure) { + publishAfterFailure(context); + throw failure; + } + publish(context); + } + + void workerFailed(Throwable failure) { + Objects.requireNonNull(failure, "failure"); + if (failure instanceof Error fatal) { + prepared.completeExceptionally(fatal); + } else if (safeFailure(failure)) { + prepared.completeExceptionally(failure); + } else { + prepared.completeExceptionally( + new MigrationOperationStoreException(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)); + } + } + + MigrationView await(Duration timeout) { + Objects.requireNonNull(timeout, "timeout"); + try { + return prepared.get(timeoutNanos(timeout), TimeUnit.NANOSECONDS); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT); + } catch (TimeoutException timeoutFailure) { + throw new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT); + } catch (ExecutionException failed) { + Throwable cause = failed.getCause(); + if (cause instanceof Error fatal) { + throw fatal; + } + if (cause instanceof MigrationOperationStoreException storeFailure) { + throw storeFailure; + } + if (cause instanceof RuntimeException stableFailure) { + throw stableFailure; + } + throw new MigrationOperationStoreException(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + } + + private static long timeoutNanos(Duration timeout) { + try { + return Math.max(1, timeout.toNanos()); + } catch (ArithmeticException overflow) { + return Long.MAX_VALUE; + } + } + + private void publishAfterFailure(RetainedCutoverPreparationContext context) { + try { + publish(context); + } catch (RuntimeException readFailure) { + prepared.completeExceptionally(readFailure); + } + } + + private void publishFatal( + RetainedCutoverPreparationContext context, Error fatal) { + try { + requireExactSnapshot(context); + } catch (RuntimeException | Error readFailure) { + fatal.addSuppressed(new MigrationOperationStoreException( + SetupErrorCode.CONFIG_RECOVERY_REQUIRED)); + } + prepared.completeExceptionally(fatal); + } + + private void publish(RetainedCutoverPreparationContext context) { + prepared.complete(MigrationOperationProjection.view(requireExactSnapshot(context))); + } + + private MigrationOperationSnapshot requireExactSnapshot( + RetainedCutoverPreparationContext context) { + DurableCutoverDraft boundDraft = requireDraft(); + MigrationOperationSnapshot snapshot = store.selectForStartup(boundDraft.operationId()) + .orElseThrow(() -> new MigrationOperationStoreException( + SetupErrorCode.OPERATION_NOT_FOUND)); + if (!exact(boundDraft, snapshot, context)) { + throw new MigrationOperationStoreException(SetupErrorCode.OPERATION_CONFLICT); + } + return store.confirmExactForStartup(snapshot); + } + + private boolean exact( + DurableCutoverDraft boundDraft, + MigrationOperationSnapshot snapshot, + RetainedCutoverPreparationContext context) { + return boundDraft.operationId().equals(context.operationId()) + && boundDraft.operationId().equals(snapshot.operationId()) + && boundDraft.target() == snapshot.target() + && boundDraft.applyMode() == snapshot.applyMode() + && boundDraft.createdAt().equals(snapshot.createdAt()) + && (snapshot.startedAt() == null || boundDraft.startedAt().equals(snapshot.startedAt())) + && Objects.equals(boundDraft.candidateGeneration(), snapshot.managedCandidateGeneration()) + && context.targetIdentityHash().equals(snapshot.targetIdentityHash()); + } + + private synchronized DurableCutoverDraft requireDraft() { + if (draft == null) { + throw new MigrationOperationStoreException(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + return draft; + } + + private synchronized RetainedCutoverPreparation requireDelegate() { + if (delegate == null) { + throw new MigrationOperationStoreException(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + return delegate; + } + + private static boolean safeFailure(Throwable failure) { + return failure instanceof MetadataMigrationException + || failure instanceof TargetJdbcConnectionException + || failure instanceof TargetSchemaProvisioningException + || failure instanceof RetainedCutoverException + || failure instanceof RetainedCutoverReleaseRequiredException + || failure instanceof RetainedCopyJournalHandoffException + || failure instanceof DurableCutoverPreparationException + || failure instanceof MigrationOperationStoreException; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationProgressJournal.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationProgressJournal.java new file mode 100644 index 0000000000..9e532ce201 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationProgressJournal.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Persists only coarse, monotonic copy and verification progress. */ +final class MigrationProgressJournal implements MetadataMigrationProgressSink { + + private final String operationId; + private final FileMigrationOperationStore store; + + MigrationProgressJournal(String operationId, FileMigrationOperationStore store) { + this.operationId = Objects.requireNonNull(operationId, "operationId"); + this.store = Objects.requireNonNull(store, "store"); + } + + @Override + public void report(MetadataMigrationStage stage, int percent) { + Objects.requireNonNull(stage, "stage"); + if (stage == MetadataMigrationStage.COPYING) { + int safePercent = Math.max(0, Math.min(percent, 99)); + store.transformAndTransitionOrConfirmDisposition( + operationId, current -> copying(current, safePercent)); + } else if (stage == MetadataMigrationStage.VERIFYING) { + store.transformAndTransitionOrConfirmDisposition(operationId, this::verifying); + } + } + + private MigrationOperationSnapshot copying(MigrationOperationSnapshot current, int percent) { + requireRunning(current, MigrationStage.COPYING); + if (percent <= current.progressPercent()) { + return current; + } + return replace(current, MigrationStage.COPYING, percent, VerificationState.PENDING); + } + + private MigrationOperationSnapshot verifying(MigrationOperationSnapshot current) { + if (current.state() == MigrationOperationState.RUNNING + && current.stage() == MigrationStage.VERIFYING) { + return current; + } + requireRunning(current, MigrationStage.COPYING); + return replace(current, MigrationStage.VERIFYING, 100, VerificationState.RUNNING); + } + + private MigrationOperationSnapshot replace( + MigrationOperationSnapshot current, + MigrationStage stage, + int progress, + VerificationState verification) { + return new MigrationOperationSnapshot( + current.operationId(), current.state(), current.target(), current.applyMode(), + stage, progress, current.createdAt(), current.startedAt(), null, verification, + null, null, current.nextPollAfterMillis(), false, false, false, + current.targetIdentityHash(), current.managedCandidateGeneration()); + } + + private static void requireRunning( + MigrationOperationSnapshot current, MigrationStage expectedStage) { + if (current.state() != MigrationOperationState.RUNNING || current.stage() != expectedStage) { + throw new MigrationOperationStoreException(SetupErrorCode.OPERATION_CONFLICT); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationTargetRequest.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationTargetRequest.java new file mode 100644 index 0000000000..7511bb8f9d --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationTargetRequest.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; + +/** Secret-free command request values retained for same-operation admission checks. */ +record MigrationTargetRequest( + String operationId, + MigrationTarget target, + ApplyMode applyMode, + MetadataDatabaseSettings settings) { + + MigrationTargetRequest { + Objects.requireNonNull(operationId, "operationId"); + Objects.requireNonNull(target, "target"); + Objects.requireNonNull(applyMode, "applyMode"); + Objects.requireNonNull(settings, "settings"); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java index 7bb3ee6326..d7fc29eb36 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java @@ -118,6 +118,11 @@ final class RetainedCutoverCoordinator { return runHandoff(state.claimPendingHandoff(operationId)); } + RetainedCutoverRecoveryPhase recoveryPhase(String operationId) { + requireOperationId(operationId); + return state.recoveryPhase(operationId); + } + RetainedManagedActivationResult activateRetained( String operationId, RetainedManagedActivation activation) { requireOperationId(operationId); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverRecoveryPhase.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverRecoveryPhase.java new file mode 100644 index 0000000000..0e95782e5b --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverRecoveryPhase.java @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +/** Secret-free exact recovery ownership retained by the cutover coordinator. */ +enum RetainedCutoverRecoveryPhase { + NONE, + FAILURE_FINALIZATION_PENDING, + RELEASE_PENDING, + HANDOFF_PENDING +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverState.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverState.java index bc2c033f7a..19fca811b5 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverState.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverState.java @@ -138,6 +138,20 @@ final class RetainedCutoverState { } } + synchronized RetainedCutoverRecoveryPhase recoveryPhase(String operationId) { + if (active == null) { + return RetainedCutoverRecoveryPhase.NONE; + } + if (!active.operationId.equals(operationId)) { + throw MigrationMaintenanceException.operationConflict(); + } + return switch (active.phase) { + case RELEASE_PENDING -> RetainedCutoverRecoveryPhase.RELEASE_PENDING; + case HANDOFF_PENDING -> RetainedCutoverRecoveryPhase.HANDOFF_PENDING; + default -> RetainedCutoverRecoveryPhase.NONE; + }; + } + private Execution require(String operationId, Phase phase) { Execution execution = requireActive(operationId); requirePhase(execution, phase); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationCommandRunnerFailureTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationCommandRunnerFailureTest.java new file mode 100644 index 0000000000..4508b92d72 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationCommandRunnerFailureTest.java @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.nio.file.Path; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceException; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MetadataMigrationRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.CandidateRef; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.MetadataTargetStageResult; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.StageOutcome; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; + +@Timeout(15) +class DeploymentMigrationCommandRunnerFailureTest { + + private static final String OPERATION = "operation-a"; + private static final String IDENTITY = "b".repeat(64); + private static final Instant NOW = Instant.parse("2026-08-10T05:00:00Z"); + + @TempDir + private Path root; + + @Test + void knownCleanupCopyAndVerificationFailuresBecomeDurableTerminalStates() throws Exception { + assertKnownFailure(MetadataMigrationStage.COPYING, 43, + MetadataMigrationErrorCode.COPY, SetupErrorCode.MIGRATION_COPY_FAILED, 43); + assertKnownFailure(MetadataMigrationStage.COPYING, 43, + MetadataMigrationErrorCode.SCHEMA, SetupErrorCode.MIGRATION_COPY_FAILED, 43); + assertKnownFailure(MetadataMigrationStage.VERIFYING, 65, + MetadataMigrationErrorCode.VERIFICATION, + SetupErrorCode.MIGRATION_VERIFICATION_FAILED, 100); + assertKnownFailure(MetadataMigrationStage.VERIFYING, 65, + MetadataMigrationErrorCode.SEQUENCE, + SetupErrorCode.MIGRATION_VERIFICATION_FAILED, 100); + } + + @Test + void timeoutOutcomeUnknownReleaseRequiredHandoffAndErrorRemainNonterminal() throws Exception { + assertNonterminal(new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT)); + assertNonterminal(new MetadataMigrationException(MetadataMigrationErrorCode.COMMIT_OUTCOME_UNKNOWN)); + assertNonterminal(new MetadataMigrationException(MetadataMigrationErrorCode.ROLLBACK_OUTCOME_UNKNOWN)); + assertNonterminal(new RetainedCutoverReleaseRequiredException()); + assertNonterminal(new RetainedCopyJournalHandoffException(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)); + assertNonterminal(new TargetSchemaProvisioningException( + MetadataDatabaseKind.MYSQL, + new TargetSchemaProvisioningFailure( + TargetSchemaProvisioningFailure.Phase.PRECONDITION, "B206", null, 0))); + assertNonterminal(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.UNAVAILABLE)); + assertNonterminal(MigrationMaintenanceException.maintenanceFailure()); + assertNonterminal(new RetainedCutoverException(RetainedCutoverErrorCode.EXECUTION_FAILED)); + assertNonterminal(new AssertionError("fatal-copy")); + } + + @Test + void progressJournalIsMonotonicAndUsesOnlyCoarseCopyAndVerificationStates() { + FileMigrationOperationStore store = preparedStore(); + MigrationProgressJournal progress = new MigrationProgressJournal(OPERATION, store); + + progress.report(MetadataMigrationStage.COPYING, 12); + progress.report(MetadataMigrationStage.COPYING, 47); + progress.report(MetadataMigrationStage.COPYING, 30); + assertThat(store.find(OPERATION).orElseThrow().progressPercent()).isEqualTo(47); + + progress.report(MetadataMigrationStage.VERIFYING, 65); + MigrationOperationSnapshot verifying = store.find(OPERATION).orElseThrow(); + assertThat(verifying.stage()).isEqualTo(MigrationStage.VERIFYING); + assertThat(verifying.progressPercent()).isEqualTo(100); + assertThat(verifying.verificationState()).isEqualTo(VerificationState.RUNNING); + + progress.report(MetadataMigrationStage.REPAIRING, 90); + progress.report(MetadataMigrationStage.COMPLETE, 100); + assertThat(store.find(OPERATION)).contains(verifying); + } + + @Test + void workerRejectionClearsTheSlotAndClosesTaskSecret() { + RetainedCutoverCoordinator coordinator = mock(RetainedCutoverCoordinator.class); + ManagedMigrationConfigurationTransaction configuration = configuration(); + ExecutorService rejected = mock(ExecutorService.class); + when(rejected.isShutdown()).thenReturn(false); + when(rejected.isTerminated()).thenReturn(true); + org.mockito.Mockito.doThrow(new RejectedExecutionException("private executor detail")) + .when(rejected).execute(any()); + DeploymentMigrationCommandRunner runner = new DeploymentMigrationCommandRunner( + new FileMigrationOperationStore(root), configuration, coordinator, + Clock.fixed(NOW, ZoneOffset.UTC), Duration.ofSeconds(30), rejected); + + assertThatThrownBy(() -> runner.start(request())) + .isInstanceOfSatisfying(MigrationOperationStoreException.class, + failure -> assertThat(failure.errorCode()) + .isEqualTo(SetupErrorCode.MIGRATION_UNAVAILABLE)) + .hasNoCause() + .hasMessageNotContaining("private"); + assertThat(runner.activeOperationId()).isEmpty(); + runner.close(); + } + + @Test + void defaultWorkerIsSingleDaemonWithZeroQueue() throws Exception { + java.util.concurrent.ThreadPoolExecutor worker = MigrationCommandWorker.create(); + try { + assertThat(worker.getCorePoolSize()).isOne(); + assertThat(worker.getMaximumPoolSize()).isOne(); + assertThat(worker.getQueue()).isInstanceOf(java.util.concurrent.SynchronousQueue.class); + CountDownLatch observed = new CountDownLatch(1); + java.util.concurrent.atomic.AtomicBoolean daemon = new java.util.concurrent.atomic.AtomicBoolean(); + worker.execute(() -> { + daemon.set(Thread.currentThread().isDaemon()); + observed.countDown(); + }); + assertThat(observed.await(5, SECONDS)).isTrue(); + assertThat(daemon).isTrue(); + } finally { + worker.shutdownNow(); + } + } + + @Test + void closeRejectsNewCommandsWaitsForWorkerAndDoesNotReleaseRetainedFence() throws Exception { + FileMigrationOperationStore store = new FileMigrationOperationStore(root.resolve("close")); + ManagedMigrationConfigurationTransaction configuration = configuration(); + RetainedCutoverCoordinator coordinator = mock(RetainedCutoverCoordinator.class); + CountDownLatch releaseCopy = new CountDownLatch(1); + when(coordinator.execute(any(), any(), any(), any(), any(), any(), any())) + .thenAnswer(invocation -> { + RetainedCutoverPreparation preparation = invocation.getArgument(5); + preparation.prepare( + new RetainedCutoverPreparationContext(OPERATION, IDENTITY), + invocation.getArgument(1), invocation.getArgument(2)); + assertThat(releaseCopy.await(5, SECONDS)).isTrue(); + return new RetainedCutoverResult( + OPERATION, IDENTITY, RetainedCutoverResult.Status.RETAINED_SUCCESS); + }); + DeploymentMigrationCommandRunner runner = new DeploymentMigrationCommandRunner( + store, configuration, coordinator, Clock.fixed(NOW, ZoneOffset.UTC), + Duration.ofSeconds(30), Executors.newSingleThreadExecutor()); + runner.start(request()); + Thread closer = Thread.ofPlatform().unstarted(runner::close); + closer.start(); + try { + awaitWaiting(closer); + assertThatThrownBy(() -> runner.start(request())) + .isInstanceOfSatisfying(MigrationOperationStoreException.class, + failure -> assertThat(failure.errorCode()) + .isEqualTo(SetupErrorCode.MIGRATION_UNAVAILABLE)); + assertThat(closer.isAlive()).isTrue(); + verify(coordinator, never()).releaseRetained(any()); + } finally { + releaseCopy.countDown(); + closer.join(5000); + } + assertThat(closer.isAlive()).isFalse(); + } + + private void assertKnownFailure( + MetadataMigrationStage stage, + int percent, + MetadataMigrationErrorCode copyFailure, + SetupErrorCode expected, + int expectedProgress) throws Exception { + Path caseRoot = root.resolve(copyFailure.name()); + Fixture fixture = fixture(caseRoot, new MetadataMigrationException(copyFailure), stage, percent); + try (DeploymentMigrationCommandRunner runner = fixture.runner()) { + runner.start(request()); + assertThat(fixture.workerDone.await(5, SECONDS)).isTrue(); + } + MigrationOperationSnapshot terminal = fixture.store.find(OPERATION).orElseThrow(); + assertThat(terminal.state()).isEqualTo(MigrationOperationState.FAILED); + assertThat(terminal.errorCode()).isEqualTo(expected); + assertThat(terminal.progressPercent()).isEqualTo(expectedProgress); + } + + private void assertNonterminal(Throwable failure) throws Exception { + Path caseRoot = root.resolve(failure.getClass().getSimpleName() + + (failure instanceof MetadataMigrationException metadata ? metadata.code().name() : "")); + Fixture fixture = fixture(caseRoot, failure, MetadataMigrationStage.COPYING, 31); + try (DeploymentMigrationCommandRunner runner = fixture.runner()) { + runner.start(request()); + assertThat(fixture.workerDone.await(5, SECONDS)).isTrue(); + } + MigrationOperationSnapshot current = fixture.store.find(OPERATION).orElseThrow(); + assertThat(current.state()).isEqualTo(MigrationOperationState.RUNNING); + assertThat(current.stage()).isEqualTo(MigrationStage.COPYING); + assertThat(current.progressPercent()).isEqualTo(31); + } + + private Fixture fixture( + Path caseRoot, Throwable failure, MetadataMigrationStage stage, int percent) { + FileMigrationOperationStore store = new FileMigrationOperationStore(caseRoot); + ManagedMigrationConfigurationTransaction configuration = configuration(); + RetainedCutoverCoordinator coordinator = mock(RetainedCutoverCoordinator.class); + CountDownLatch workerDone = new CountDownLatch(1); + when(coordinator.execute(any(), any(), any(), any(), any(), any(), any())) + .thenAnswer(invocation -> { + try { + RetainedCutoverPreparation preparation = invocation.getArgument(5); + preparation.prepare( + new RetainedCutoverPreparationContext(OPERATION, IDENTITY), + invocation.getArgument(1), invocation.getArgument(2)); + MetadataMigrationProgressSink progress = invocation.getArgument(4); + progress.report(stage, percent); + if (failure instanceof Error error) { + throw error; + } + throw (RuntimeException) failure; + } finally { + workerDone.countDown(); + } + }); + return new Fixture(store, configuration, coordinator, workerDone); + } + + private ManagedMigrationConfigurationTransaction configuration() { + ManagedMigrationConfigurationTransaction configuration = mock( + ManagedMigrationConfigurationTransaction.class); + try { + when(configuration.stageMetadataTarget(any(), any(), any(), any(), any())) + .thenAnswer(invocation -> new MetadataTargetStageResult( + StageOutcome.STAGED, + Optional.of(new CandidateRef( + invocation.getArgument(0), invocation.getArgument(1))))); + } catch (java.io.IOException impossible) { + throw new AssertionError(impossible); + } + return configuration; + } + + private FileMigrationOperationStore preparedStore() { + FileMigrationOperationStore store = new FileMigrationOperationStore(root.resolve("progress")); + Instant created = NOW; + String generation = MigrationCandidateGeneration.fromOperationId(OPERATION); + MigrationOperationSnapshot pending = new MigrationOperationSnapshot( + OPERATION, MigrationOperationState.PENDING, MigrationTarget.MYSQL, + ApplyMode.MANAGED_WRITE, MigrationStage.QUEUED, 0, created, null, null, + VerificationState.PENDING, null, null, 1000, false, false, false, + IDENTITY, generation); + MigrationOperationSnapshot running = new MigrationOperationSnapshot( + OPERATION, MigrationOperationState.RUNNING, MigrationTarget.MYSQL, + ApplyMode.MANAGED_WRITE, MigrationStage.COPYING, 0, created, created, null, + VerificationState.PENDING, null, null, 1000, false, false, false, + IDENTITY, generation); + store.create(pending); + store.compareAndTransition(OPERATION, MigrationOperationState.PENDING, running); + return store; + } + + private record Fixture( + FileMigrationOperationStore store, + ManagedMigrationConfigurationTransaction configuration, + RetainedCutoverCoordinator coordinator, + CountDownLatch workerDone) { + + DeploymentMigrationCommandRunner runner() { + return new DeploymentMigrationCommandRunner( + store, configuration, coordinator, Clock.fixed(NOW, ZoneOffset.UTC), + Duration.ofSeconds(30), Executors.newSingleThreadExecutor()); + } + } + + private static MetadataMigrationRequest request() { + return new MetadataMigrationRequest( + OPERATION, MigrationTarget.MYSQL, + new MetadataDatabaseConfiguration( + MetadataDatabaseKind.MYSQL, + "jdbc:mysql://db.example/hertzbeat", "migration", "password-a"), + ApplyMode.MANAGED_WRITE); + } + + private static void awaitWaiting(Thread thread) { + long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(5); + while (thread.getState() != Thread.State.WAITING + && thread.getState() != Thread.State.TIMED_WAITING + && System.nanoTime() < deadline) { + Thread.onSpinWait(); + } + assertThat(thread.getState()).isIn(Thread.State.WAITING, Thread.State.TIMED_WAITING); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationCommandRunnerRecoveryTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationCommandRunnerRecoveryTest.java new file mode 100644 index 0000000000..0dc198b6d3 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationCommandRunnerRecoveryTest.java @@ -0,0 +1,487 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.nio.file.Path; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MetadataMigrationRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationView; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.CandidateRef; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.MetadataTargetStageResult; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.StageOutcome; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.invocation.InvocationOnMock; + +@Timeout(15) +class DeploymentMigrationCommandRunnerRecoveryTest { + + private static final String OPERATION = "operation-a"; + private static final String IDENTITY = "c".repeat(64); + private static final Instant NOW = Instant.parse("2026-08-10T07:00:00Z"); + + @TempDir + private Path root; + + @Test + void preparationWaitIsBoundedWithoutCancellingWorkerAndLaterJoinDoesNotRecopy() throws Exception { + FileMigrationOperationStore store = new FileMigrationOperationStore(root.resolve("timeout")); + RetainedCutoverCoordinator coordinator = mock(RetainedCutoverCoordinator.class); + CountDownLatch allowPreparation = new CountDownLatch(1); + AtomicInteger executions = new AtomicInteger(); + when(coordinator.execute(any(), any(), any(), any(), any(), any(), any())) + .thenAnswer(invocation -> { + executions.incrementAndGet(); + assertThat(allowPreparation.await(5, SECONDS)).isTrue(); + prepare(invocation); + return retained(); + }); + DeploymentMigrationCommandRunner runner = runner( + store, coordinator, Duration.ofMillis(100)); + try { + assertThatThrownBy(() -> runner.start(request())) + .isInstanceOfSatisfying(MetadataMigrationException.class, + failure -> assertThat(failure.code()) + .isEqualTo(MetadataMigrationErrorCode.TIMEOUT)); + assertThat(runner.activeOperationId()).contains(OPERATION); + + allowPreparation.countDown(); + MigrationViewAssert.running(runner.start(request())); + assertThat(executions).hasValue(1); + } finally { + allowPreparation.countDown(); + runner.close(); + } + } + + @Test + void releaseAndHandoffPendingRetrySameOperationWithoutRecopy() throws Exception { + assertRetry(RetainedCutoverRecoveryPhase.RELEASE_PENDING); + assertRetry(RetainedCutoverRecoveryPhase.HANDOFF_PENDING); + } + + @Test + void interruptedHandoffSubmissionPreservesTimeoutAndExactPendingOwnership() throws Exception { + FileMigrationOperationStore store = new FileMigrationOperationStore(root.resolve("interrupted-handoff")); + RetainedCutoverCoordinator coordinator = pendingCoordinator( + RetainedCutoverRecoveryPhase.HANDOFF_PENDING); + when(coordinator.recoveryPhase(OPERATION)).thenReturn( + RetainedCutoverRecoveryPhase.HANDOFF_PENDING, + RetainedCutoverRecoveryPhase.NONE); + when(coordinator.retryHandoff(OPERATION)).thenReturn(retained()); + CountDownLatch workerUnwinding = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + ThreadPoolExecutor worker = blockingAfterExecute(workerUnwinding, releaseWorker); + DeploymentMigrationCommandRunner runner = new DeploymentMigrationCommandRunner( + store, configuration(), coordinator, Clock.fixed(NOW, ZoneOffset.UTC), + Duration.ofSeconds(2), worker); + try { + MigrationViewAssert.running(runner.start(request())); + assertThat(workerUnwinding.await(5, SECONDS)).isTrue(); + Thread.currentThread().interrupt(); + try { + assertThatThrownBy(() -> runner.start(request())) + .isInstanceOfSatisfying(MetadataMigrationException.class, + failure -> assertThat(failure.code()) + .isEqualTo(MetadataMigrationErrorCode.TIMEOUT)); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } finally { + Thread.interrupted(); + } + assertThat(runner.activeRecoveryPhase()) + .contains(RetainedCutoverRecoveryPhase.HANDOFF_PENDING); + + releaseWorker.countDown(); + MigrationViewAssert.running(runner.start(request())); + } finally { + releaseWorker.countDown(); + Thread.interrupted(); + runner.close(); + } + verify(coordinator).retryHandoff(OPERATION); + verify(coordinator).execute(any(), any(), any(), any(), any(), any(), any()); + } + + @Test + void closePreservesPendingReleaseForExactRetryAndFatalMarkerUsesSameOwnership() throws Exception { + FileMigrationOperationStore retryStore = new FileMigrationOperationStore(root.resolve("close-retry")); + RetainedCutoverCoordinator retryCoordinator = pendingCoordinator( + RetainedCutoverRecoveryPhase.RELEASE_PENDING); + when(retryCoordinator.recoveryPhase(OPERATION)).thenReturn( + RetainedCutoverRecoveryPhase.RELEASE_PENDING, + RetainedCutoverRecoveryPhase.RELEASE_PENDING, + RetainedCutoverRecoveryPhase.NONE); + when(retryCoordinator.retryRelease(OPERATION, Duration.ofSeconds(2))) + .thenThrow(new RetainedCutoverReleaseRequiredException()) + .thenReturn(retained()); + DeploymentMigrationCommandRunner retryRunner = runner( + retryStore, retryCoordinator, Duration.ofSeconds(2)); + retryRunner.start(request()); + awaitRecovery(retryRunner, RetainedCutoverRecoveryPhase.RELEASE_PENDING); + assertThatThrownBy(retryRunner::close) + .isInstanceOf(RetainedCutoverReleaseRequiredException.class); + assertThat(retryRunner.activeOperationId()).contains(OPERATION); + retryRunner.close(); + verify(retryCoordinator, times(2)).retryRelease(OPERATION, Duration.ofSeconds(2)); + + FileMigrationOperationStore fatalStore = new FileMigrationOperationStore(root.resolve("fatal")); + RetainedCutoverCoordinator fatalCoordinator = mock(RetainedCutoverCoordinator.class); + AssertionError fatal = new AssertionError("fatal-copy"); + RetainedCutoverReleaseRequiredException.attach(fatal); + when(fatalCoordinator.execute(any(), any(), any(), any(), any(), any(), any())) + .thenAnswer(invocation -> { + prepare(invocation); + throw fatal; + }); + when(fatalCoordinator.recoveryPhase(OPERATION)).thenReturn( + RetainedCutoverRecoveryPhase.RELEASE_PENDING, + RetainedCutoverRecoveryPhase.NONE); + when(fatalCoordinator.retryRelease(OPERATION, Duration.ofSeconds(2))).thenThrow(fatal); + DeploymentMigrationCommandRunner fatalRunner = runner( + fatalStore, fatalCoordinator, Duration.ofSeconds(2)); + fatalRunner.start(request()); + awaitRecovery(fatalRunner, RetainedCutoverRecoveryPhase.RELEASE_PENDING); + assertThatThrownBy(fatalRunner::close).isSameAs(fatal); + assertThat(fatalRunner.activeRecoveryPhase()).isEmpty(); + fatalRunner.close(); + } + + @Test + void swallowedVerifyingProgressFailureUsesAuthoritativeCopyingStage() throws Exception { + Path caseRoot = root.resolve("progress-failure"); + MigrationOperationFilePublisher publisher = new MigrationOperationFilePublisher(caseRoot); + AtomicBoolean failNext = new AtomicBoolean(); + FileMigrationOperationStore store = new FileMigrationOperationStore(caseRoot, (target, content) -> { + if (failNext.compareAndSet(true, false)) { + throw new IOException("private-progress-failure"); + } + publisher.publish(target, content); + }); + RetainedCutoverCoordinator coordinator = mock(RetainedCutoverCoordinator.class); + when(coordinator.execute(any(), any(), any(), any(), any(), any(), any())) + .thenAnswer(invocation -> { + prepare(invocation); + failNext.set(true); + MetadataMigrationProgressSink progress = invocation.getArgument(4); + try { + progress.report(MetadataMigrationStage.VERIFYING, 100); + } catch (MigrationOperationStoreException swallowedByJdbcBoundary) { + assertThat(swallowedByJdbcBoundary.errorCode()) + .isEqualTo(SetupErrorCode.CONFIG_WRITE_FAILED); + } + throw new MetadataMigrationException(MetadataMigrationErrorCode.VERIFICATION); + }); + + try (DeploymentMigrationCommandRunner runner = runner( + store, coordinator, Duration.ofSeconds(2))) { + MigrationViewAssert.running(runner.start(request())); + } + MigrationOperationSnapshot terminal = store.find(OPERATION).orElseThrow(); + assertThat(terminal.state()).isEqualTo(MigrationOperationState.FAILED); + assertThat(terminal.stage()).isEqualTo(MigrationStage.FAILED); + assertThat(terminal.errorCode()).isEqualTo(SetupErrorCode.MIGRATION_COPY_FAILED); + } + + @Test + void findCannotPublishReadyWhileHandoffFailureStillOwnsTheTask() throws Exception { + Path caseRoot = root.resolve("find-handoff-race"); + MigrationOperationFilePublisher publisher = new MigrationOperationFilePublisher(caseRoot); + AtomicBoolean failFinalHandoff = new AtomicBoolean(); + AtomicInteger handoffWrites = new AtomicInteger(); + CountDownLatch readyCommitted = new CountDownLatch(1); + CountDownLatch releasePublisher = new CountDownLatch(1); + FileMigrationOperationStore store = new FileMigrationOperationStore(caseRoot, (target, content) -> { + publisher.publish(target, content); + if (failFinalHandoff.get() && handoffWrites.incrementAndGet() == 2) { + readyCommitted.countDown(); + try { + assertThat(releasePublisher.await(5, SECONDS)).isTrue(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IOException("test publisher interrupted"); + } + throw new IOException("private final handoff failure"); + } + }); + RetainedCutoverCoordinator coordinator = mock(RetainedCutoverCoordinator.class); + AtomicInteger copies = new AtomicInteger(); + when(coordinator.execute(any(), any(), any(), any(), any(), any(), any())) + .thenAnswer(invocation -> { + copies.incrementAndGet(); + prepare(invocation); + MetadataMigrationProgressSink progress = invocation.getArgument(4); + progress.report(MetadataMigrationStage.VERIFYING, 100); + failFinalHandoff.set(true); + RetainedCopyJournalHandoff handoff = invocation.getArgument(6); + return handoff.handoff(new RetainedCopyJournalContext(OPERATION, IDENTITY)); + }); + when(coordinator.retryHandoff(OPERATION)).thenAnswer(ignored -> { + MigrationOperationSnapshot current = store.find(OPERATION).orElseThrow(); + store.confirmExactForStartup(current); + return retained(); + }); + DeploymentMigrationCommandRunner runner = runner( + store, coordinator, Duration.ofSeconds(2)); + AtomicReference> projected = new AtomicReference<>(); + AtomicReference projectionFailure = new AtomicReference<>(); + Thread find = null; + try { + MigrationViewAssert.running(runner.start(request())); + assertThat(readyCommitted.await(5, SECONDS)).isTrue(); + find = Thread.ofPlatform().start(() -> { + try { + projected.set(runner.find(OPERATION)); + } catch (RuntimeException | Error failure) { + projectionFailure.set(failure); + } + }); + awaitState(find, Thread.State.WAITING); + releasePublisher.countDown(); + find.join(5000); + + assertThat(projected.get()).isNull(); + assertThat(projectionFailure.get()) + .isInstanceOfSatisfying(MigrationOperationStoreException.class, + failure -> assertThat(failure.errorCode()) + .isEqualTo(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)); + awaitRecovery(runner, RetainedCutoverRecoveryPhase.HANDOFF_PENDING); + + MigrationViewAssert.running(runner.start(request())); + assertThat(awaitReadable(runner)).hasValueSatisfying( + view -> assertThat(view.state()) + .isEqualTo(MigrationOperationState.READY_TO_ACTIVATE)); + assertThat(copies).hasValue(1); + } finally { + releasePublisher.countDown(); + if (find != null) { + find.join(5000); + } + runner.close(); + } + } + + @Test + void owningConstructorUsesRealDaemonWorkerAndJournalFallbackFindsActiveOperation() throws Exception { + FileMigrationOperationStore store = new FileMigrationOperationStore(root.resolve("owned")); + RetainedCutoverCoordinator coordinator = mock(RetainedCutoverCoordinator.class); + AtomicBoolean daemon = new AtomicBoolean(); + when(coordinator.execute(any(), any(), any(), any(), any(), any(), any())) + .thenAnswer(invocation -> { + daemon.set(Thread.currentThread().isDaemon()); + prepare(invocation); + return retained(); + }); + try (DeploymentMigrationCommandRunner runner = new DeploymentMigrationCommandRunner( + store, configuration(), coordinator, Clock.fixed(NOW, ZoneOffset.UTC), + Duration.ofSeconds(2))) { + MigrationViewAssert.running(runner.start(request())); + } + assertThat(daemon).isTrue(); + + FileMigrationOperationStore fallbackStore = new FileMigrationOperationStore(root.resolve("fallback")); + fallbackStore.create(pending()); + try (DeploymentMigrationCommandRunner runner = new DeploymentMigrationCommandRunner( + fallbackStore, configuration(), mock(RetainedCutoverCoordinator.class), + Clock.fixed(NOW, ZoneOffset.UTC), Duration.ofSeconds(2))) { + assertThat(runner.activeOperationId()).contains(OPERATION); + } + } + + private void assertRetry(RetainedCutoverRecoveryPhase phase) throws Exception { + FileMigrationOperationStore store = new FileMigrationOperationStore(root.resolve(phase.name())); + RetainedCutoverCoordinator coordinator = pendingCoordinator(phase); + when(coordinator.recoveryPhase(OPERATION)).thenReturn(phase, RetainedCutoverRecoveryPhase.NONE); + if (phase == RetainedCutoverRecoveryPhase.RELEASE_PENDING) { + when(coordinator.retryRelease(OPERATION, Duration.ofSeconds(2))).thenReturn(retained()); + } else { + when(coordinator.retryHandoff(OPERATION)).thenReturn(retained()); + } + try (DeploymentMigrationCommandRunner runner = runner( + store, coordinator, Duration.ofSeconds(2))) { + MigrationViewAssert.running(runner.start(request())); + awaitRecovery(runner, phase); + assertThatThrownBy(() -> runner.find(OPERATION)) + .isInstanceOfSatisfying(MigrationOperationStoreException.class, + failure -> assertThat(failure.errorCode()) + .isEqualTo(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)); + MigrationViewAssert.running(runner.start(request())); + if (phase == RetainedCutoverRecoveryPhase.HANDOFF_PENDING) { + assertThat(awaitReadable(runner)).hasValueSatisfying( + view -> assertThat(view.state()) + .isEqualTo(MigrationOperationState.READY_TO_ACTIVATE)); + } + } + verify(coordinator).execute(any(), any(), any(), any(), any(), any(), any()); + if (phase == RetainedCutoverRecoveryPhase.RELEASE_PENDING) { + verify(coordinator).retryRelease(OPERATION, Duration.ofSeconds(2)); + } else { + verify(coordinator).retryHandoff(OPERATION); + } + } + + private RetainedCutoverCoordinator pendingCoordinator(RetainedCutoverRecoveryPhase phase) { + RetainedCutoverCoordinator coordinator = mock(RetainedCutoverCoordinator.class); + when(coordinator.execute(any(), any(), any(), any(), any(), any(), any())) + .thenAnswer(invocation -> { + prepare(invocation); + if (phase == RetainedCutoverRecoveryPhase.RELEASE_PENDING) { + throw new RetainedCutoverReleaseRequiredException(); + } + MetadataMigrationProgressSink progress = invocation.getArgument(4); + progress.report(MetadataMigrationStage.VERIFYING, 100); + RetainedCopyJournalHandoff handoff = invocation.getArgument(6); + handoff.handoff(new RetainedCopyJournalContext(OPERATION, IDENTITY)); + throw new RetainedCopyJournalHandoffException( + SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + }); + return coordinator; + } + + private static ThreadPoolExecutor blockingAfterExecute( + CountDownLatch entered, CountDownLatch release) { + return new ThreadPoolExecutor(1, 1, 0, MILLISECONDS, new SynchronousQueue<>()) { + @Override + protected void afterExecute(Runnable task, Throwable failure) { + entered.countDown(); + try { + assertThat(release.await(5, SECONDS)).isTrue(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + } + }; + } + + private DeploymentMigrationCommandRunner runner( + FileMigrationOperationStore store, + RetainedCutoverCoordinator coordinator, + Duration timeout) { + return new DeploymentMigrationCommandRunner( + store, configuration(), coordinator, Clock.fixed(NOW, ZoneOffset.UTC), + timeout, Executors.newSingleThreadExecutor()); + } + + private ManagedMigrationConfigurationTransaction configuration() { + ManagedMigrationConfigurationTransaction configuration = mock( + ManagedMigrationConfigurationTransaction.class); + try { + when(configuration.stageMetadataTarget(any(), any(), any(), any(), any())) + .thenAnswer(invocation -> new MetadataTargetStageResult( + StageOutcome.STAGED, + Optional.of(new CandidateRef( + invocation.getArgument(0), invocation.getArgument(1))))); + } catch (IOException impossible) { + throw new AssertionError(impossible); + } + return configuration; + } + + private static void prepare(InvocationOnMock invocation) { + RetainedCutoverPreparation preparation = invocation.getArgument(5); + preparation.prepare( + new RetainedCutoverPreparationContext(OPERATION, IDENTITY), + invocation.getArgument(1), invocation.getArgument(2)); + } + + private static RetainedCutoverResult retained() { + return new RetainedCutoverResult( + OPERATION, IDENTITY, RetainedCutoverResult.Status.RETAINED_SUCCESS); + } + + private static void awaitRecovery( + DeploymentMigrationCommandRunner runner, RetainedCutoverRecoveryPhase phase) { + long deadline = System.nanoTime() + SECONDS.toNanos(5); + while (!runner.activeRecoveryPhase().equals(Optional.of(phase)) + && System.nanoTime() < deadline) { + Thread.onSpinWait(); + } + assertThat(runner.activeRecoveryPhase()).contains(phase); + } + + private static Optional awaitReadable(DeploymentMigrationCommandRunner runner) { + long deadline = System.nanoTime() + SECONDS.toNanos(5); + while (System.nanoTime() < deadline) { + try { + return runner.find(OPERATION); + } catch (MigrationOperationStoreException pending) { + assertThat(pending.errorCode()).isEqualTo(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + Thread.onSpinWait(); + } + } + return runner.find(OPERATION); + } + + private static void awaitState(Thread thread, Thread.State expected) { + long deadline = System.nanoTime() + SECONDS.toNanos(5); + while (thread.getState() != expected && System.nanoTime() < deadline) { + Thread.onSpinWait(); + } + assertThat(thread.getState()).isEqualTo(expected); + } + + private static MetadataMigrationRequest request() { + return new MetadataMigrationRequest( + OPERATION, MigrationTarget.MYSQL, + new MetadataDatabaseConfiguration( + MetadataDatabaseKind.MYSQL, + "jdbc:mysql://db.example/hertzbeat", "migration", "password-a"), + ApplyMode.MANAGED_WRITE); + } + + private static MigrationOperationSnapshot pending() { + return new MigrationOperationSnapshot( + OPERATION, MigrationOperationState.PENDING, MigrationTarget.MYSQL, + ApplyMode.MANAGED_WRITE, MigrationStage.QUEUED, 0, NOW, null, null, + VerificationState.PENDING, + null, null, 1000, false, false, false, IDENTITY, + MigrationCandidateGeneration.fromOperationId(OPERATION)); + } + + private static final class MigrationViewAssert { + + private MigrationViewAssert() { + } + + static void running(MigrationView view) { + assertThat(view.state()).isEqualTo(MigrationOperationState.RUNNING); + assertThat(view.stage()).isEqualTo(MigrationStage.COPYING); + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationCommandRunnerReviewTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationCommandRunnerReviewTest.java new file mode 100644 index 0000000000..335bd029b0 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationCommandRunnerReviewTest.java @@ -0,0 +1,444 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.nio.file.Path; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MetadataMigrationRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.CandidateRef; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.MetadataTargetStageResult; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.StageOutcome; +import org.apache.hertzbeat.manager.setup.security.CommittedSetupFileDurabilityException; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFileLock; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.invocation.InvocationOnMock; + +@Timeout(15) +class DeploymentMigrationCommandRunnerReviewTest { + + private static final String OPERATION = "operation-a"; + private static final String IDENTITY = "d".repeat(64); + private static final Instant NOW = Instant.parse("2026-08-10T10:00:00Z"); + + @TempDir + private Path root; + + @Test + void failedFinalizationPinsOutcomeAndSameOperationRetriesWithoutRecopy() throws Exception { + Path caseRoot = root.resolve("finalization"); + MigrationOperationFilePublisher publisher = new MigrationOperationFilePublisher(caseRoot); + AtomicBoolean rejectWrites = new AtomicBoolean(); + FileMigrationOperationStore store = new FileMigrationOperationStore(caseRoot, (target, content) -> { + if (rejectWrites.get()) { + throw new IOException("private-finalization-write"); + } + publisher.publish(target, content); + }); + AtomicReference time = new AtomicReference<>(NOW); + Clock clock = mock(Clock.class); + when(clock.instant()).thenAnswer(ignored -> time.get()); + RetainedCutoverCoordinator coordinator = mock(RetainedCutoverCoordinator.class); + AtomicInteger copies = new AtomicInteger(); + when(coordinator.execute(any(), any(), any(), any(), any(), any(), any())) + .thenAnswer(invocation -> { + copies.incrementAndGet(); + prepare(invocation); + rejectWrites.set(true); + throw new MetadataMigrationException(MetadataMigrationErrorCode.COPY); + }); + + DeploymentMigrationCommandRunner runner = runner(store, coordinator, clock, + Executors.newSingleThreadExecutor()); + try { + runner.start(request()); + awaitPhase(runner, RetainedCutoverRecoveryPhase.FAILURE_FINALIZATION_PENDING); + time.set(NOW.plusSeconds(30)); + rejectWrites.set(false); + + runner.start(request()); + awaitTerminal(store); + MigrationOperationSnapshot failed = store.find(OPERATION).orElseThrow(); + assertThat(failed.errorCode()).isEqualTo(SetupErrorCode.MIGRATION_COPY_FAILED); + assertThat(failed.completedAt()).isEqualTo(NOW); + assertThat(copies).hasValue(1); + } finally { + rejectWrites.set(false); + runner.close(); + } + } + + @Test + void committedFinalizationRemainsOwnedUntilCloseConfirmsTheExactTerminalRecord() { + Path caseRoot = root.resolve("committed-finalization"); + MigrationOperationFilePublisher publisher = new MigrationOperationFilePublisher(caseRoot); + AtomicBoolean uncertain = new AtomicBoolean(); + FileMigrationOperationStore store = new FileMigrationOperationStore(caseRoot, (target, content) -> { + publisher.publish(target, content); + if (uncertain.get()) { + throw new CommittedSetupFileDurabilityException(); + } + }); + RetainedCutoverCoordinator coordinator = mock(RetainedCutoverCoordinator.class); + AtomicInteger copies = new AtomicInteger(); + when(coordinator.execute(any(), any(), any(), any(), any(), any(), any())) + .thenAnswer(invocation -> { + copies.incrementAndGet(); + prepare(invocation); + uncertain.set(true); + throw new MetadataMigrationException(MetadataMigrationErrorCode.COPY); + }); + + DeploymentMigrationCommandRunner runner = runner( + store, coordinator, Clock.fixed(NOW, ZoneOffset.UTC), + Executors.newSingleThreadExecutor()); + runner.start(request()); + awaitPhase(runner, RetainedCutoverRecoveryPhase.FAILURE_FINALIZATION_PENDING); + assertThat(store.find(OPERATION)).hasValueSatisfying( + snapshot -> assertThat(snapshot.errorCode()) + .isEqualTo(SetupErrorCode.MIGRATION_COPY_FAILED)); + assertThatThrownBy(() -> runner.find(OPERATION)) + .isInstanceOfSatisfying(MigrationOperationStoreException.class, + failure -> assertThat(failure.errorCode()) + .isEqualTo(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)); + uncertain.set(false); + runner.close(); + + assertThat(copies).hasValue(1); + assertThat(store.find(OPERATION)).hasValueSatisfying( + snapshot -> assertThat(snapshot.completedAt()).isEqualTo(NOW)); + } + + @Test + void recoveryProbeFatalCannotHideOriginalFatalOrSkipOwnerCompletion() throws Exception { + FileMigrationOperationStore store = new FileMigrationOperationStore(root.resolve("probe-fatal")); + RetainedCutoverCoordinator coordinator = mock(RetainedCutoverCoordinator.class); + AssertionError primary = new AssertionError("primary-copy-fatal"); + AssertionError probe = new AssertionError("private-recovery-probe"); + when(coordinator.execute(any(), any(), any(), any(), any(), any(), any())) + .thenAnswer(invocation -> { + prepare(invocation); + throw primary; + }); + when(coordinator.recoveryPhase(OPERATION)).thenThrow(probe); + CountDownLatch uncaught = new CountDownLatch(1); + AtomicReference observed = new AtomicReference<>(); + ExecutorService worker = Executors.newSingleThreadExecutor(task -> { + Thread thread = new Thread(task); + thread.setUncaughtExceptionHandler((ignored, failure) -> { + observed.set(failure); + uncaught.countDown(); + }); + return thread; + }); + + DeploymentMigrationCommandRunner runner = runner( + store, coordinator, Clock.fixed(NOW, ZoneOffset.UTC), worker); + runner.start(request()); + assertThat(uncaught.await(5, SECONDS)).isTrue(); + assertThat(observed.get()).isSameAs(primary); + assertThat(primary.getSuppressed()) + .singleElement() + .isInstanceOfSatisfying(MigrationOperationStoreException.class, + marker -> assertThat(marker.errorCode()) + .isEqualTo(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)); + assertThat(runner.activeRecoveryPhase()).isEmpty(); + runner.close(); + } + + @Test + void retryWaitsForZeroQueueWorkerToFinishAfterExecute() throws Exception { + FileMigrationOperationStore store = new FileMigrationOperationStore(root.resolve("handoff")); + RetainedCutoverCoordinator coordinator = mock(RetainedCutoverCoordinator.class); + CountDownLatch afterExecute = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + ThreadPoolExecutor worker = blockingAfterExecute(afterExecute, releaseWorker); + when(coordinator.execute(any(), any(), any(), any(), any(), any(), any())) + .thenAnswer(invocation -> { + prepare(invocation); + throw new RetainedCutoverReleaseRequiredException(); + }); + when(coordinator.recoveryPhase(OPERATION)).thenReturn( + RetainedCutoverRecoveryPhase.RELEASE_PENDING, + RetainedCutoverRecoveryPhase.NONE); + when(coordinator.retryRelease(OPERATION, Duration.ofSeconds(2))).thenReturn(retained()); + DeploymentMigrationCommandRunner runner = runner( + store, coordinator, Clock.fixed(NOW, ZoneOffset.UTC), worker); + try { + runner.start(request()); + assertThat(afterExecute.await(5, SECONDS)).isTrue(); + CompletableFuture retry = CompletableFuture.runAsync(() -> runner.start(request())); + assertThatThrownBy(() -> retry.get(100, MILLISECONDS)) + .isInstanceOf(java.util.concurrent.TimeoutException.class); + releaseWorker.countDown(); + retry.get(5, SECONDS); + verify(coordinator).retryRelease(OPERATION, Duration.ofSeconds(2)); + verify(coordinator).execute(any(), any(), any(), any(), any(), any(), any()); + } finally { + releaseWorker.countDown(); + runner.close(); + } + } + + @Test + void expiredWorkerHandoffRestoresExactPendingOwnershipForLaterRetry() throws Exception { + FileMigrationOperationStore store = new FileMigrationOperationStore(root.resolve("handoff-expired")); + RetainedCutoverCoordinator coordinator = mock(RetainedCutoverCoordinator.class); + CountDownLatch afterExecute = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + ThreadPoolExecutor worker = blockingAfterExecute(afterExecute, releaseWorker); + when(coordinator.execute(any(), any(), any(), any(), any(), any(), any())) + .thenAnswer(invocation -> { + prepare(invocation); + throw new RetainedCutoverReleaseRequiredException(); + }); + when(coordinator.recoveryPhase(OPERATION)).thenReturn( + RetainedCutoverRecoveryPhase.RELEASE_PENDING, + RetainedCutoverRecoveryPhase.NONE); + when(coordinator.retryRelease(OPERATION, Duration.ofMillis(100))).thenReturn(retained()); + DeploymentMigrationCommandRunner runner = new DeploymentMigrationCommandRunner( + store, configuration(), coordinator, Clock.fixed(NOW, ZoneOffset.UTC), + Duration.ofMillis(100), worker); + try { + runner.start(request()); + assertThat(afterExecute.await(5, SECONDS)).isTrue(); + assertThatThrownBy(() -> runner.start(request())) + .isInstanceOfSatisfying(MigrationOperationStoreException.class, + failure -> assertThat(failure.errorCode()) + .isEqualTo(SetupErrorCode.MIGRATION_UNAVAILABLE)); + assertThat(runner.activeRecoveryPhase()) + .contains(RetainedCutoverRecoveryPhase.RELEASE_PENDING); + + releaseWorker.countDown(); + runner.start(request()); + verify(coordinator).retryRelease(OPERATION, Duration.ofMillis(100)); + verify(coordinator).execute(any(), any(), any(), any(), any(), any(), any()); + } finally { + releaseWorker.countDown(); + runner.close(); + } + } + + @Test + void committedPreparationAndPersistedReplayRequireHealthyDurabilityConfirmation() { + Path caseRoot = root.resolve("preparation-confirm"); + MigrationOperationFilePublisher publisher = new MigrationOperationFilePublisher(caseRoot); + AtomicBoolean uncertain = new AtomicBoolean(); + FileMigrationOperationStore store = new FileMigrationOperationStore(caseRoot, (target, content) -> { + publisher.publish(target, content); + if (uncertain.get()) { + throw new CommittedSetupFileDurabilityException(); + } + }); + ManagedMigrationConfigurationTransaction configuration = configuration(); + try { + org.mockito.Mockito.doAnswer(invocation -> { + uncertain.set(true); + return stage(invocation); + }).when(configuration).stageMetadataTarget(any(), any(), any(), any(), any()); + } catch (IOException impossible) { + throw new AssertionError(impossible); + } + RetainedCutoverCoordinator coordinator = mock(RetainedCutoverCoordinator.class); + when(coordinator.execute(any(), any(), any(), any(), any(), any(), any())) + .thenAnswer(invocation -> { + prepare(invocation); + throw new AssertionError("copy-must-not-start"); + }); + DeploymentMigrationCommandRunner uncertainRunner = new DeploymentMigrationCommandRunner( + store, configuration, coordinator, Clock.fixed(NOW, ZoneOffset.UTC), + Duration.ofSeconds(2), Executors.newSingleThreadExecutor()); + assertThatThrownBy(() -> uncertainRunner.start(request())) + .isInstanceOfSatisfying(MigrationOperationStoreException.class, + failure -> assertThat(failure.errorCode()) + .isEqualTo(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)); + uncertain.set(false); + uncertainRunner.close(); + + RetainedCutoverCoordinator healthyCoordinator = mock(RetainedCutoverCoordinator.class); + try (DeploymentMigrationCommandRunner healthy = runner( + store, healthyCoordinator, Clock.fixed(NOW, ZoneOffset.UTC), + Executors.newSingleThreadExecutor())) { + assertThat(healthy.start(request()).state()).isEqualTo(MigrationOperationState.RUNNING); + } + verify(healthyCoordinator, times(0)).execute(any(), any(), any(), any(), any(), any(), any()); + } + + @Test + void activeOperationProjectionCannotMissTaskCreatedBeforeJournalWrite() throws Exception { + Path caseRoot = root.resolve("active-linearized"); + FileMigrationOperationStore store = new FileMigrationOperationStore(caseRoot); + SecureSetupFileLock fileLock = new SecureSetupFileLock( + caseRoot, "data/config/.metadata-migration-operations.lock"); + CountDownLatch lockHeld = new CountDownLatch(1); + CountDownLatch releaseLock = new CountDownLatch(1); + CompletableFuture holder = CompletableFuture.runAsync(() -> { + try { + fileLock.execute(() -> { + lockHeld.countDown(); + try { + assertThat(releaseLock.await(5, SECONDS)).isTrue(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IOException("test lock interrupted"); + } + return null; + }); + } catch (IOException failure) { + throw new AssertionError(failure); + } + }); + assertThat(lockHeld.await(5, SECONDS)).isTrue(); + DeploymentMigrationCommandRunner runner = runner( + store, mock(RetainedCutoverCoordinator.class), Clock.fixed(NOW, ZoneOffset.UTC), + Executors.newSingleThreadExecutor()); + AtomicReference> projected = new AtomicReference<>(); + Thread projection = Thread.ofPlatform().start( + () -> projected.set(runner.activeOperationId())); + AtomicReference startFailure = new AtomicReference<>(); + Thread start = Thread.ofPlatform().start(() -> { + try { + runner.start(request()); + } catch (RuntimeException | Error failure) { + startFailure.set(failure); + } + }); + try { + awaitState(start, Thread.State.BLOCKED); + releaseLock.countDown(); + projection.join(5000); + assertThat(projected.get()).isEmpty(); + } finally { + releaseLock.countDown(); + holder.get(5, SECONDS); + start.join(5000); + assertThat(startFailure.get()).isInstanceOf(MetadataMigrationException.class); + runner.close(); + } + } + + private DeploymentMigrationCommandRunner runner( + FileMigrationOperationStore store, + RetainedCutoverCoordinator coordinator, + Clock clock, + ExecutorService worker) { + return new DeploymentMigrationCommandRunner( + store, configuration(), coordinator, clock, Duration.ofSeconds(2), worker); + } + + private static ThreadPoolExecutor blockingAfterExecute( + CountDownLatch afterExecute, CountDownLatch releaseWorker) { + return new ThreadPoolExecutor(1, 1, 0, MILLISECONDS, new SynchronousQueue<>()) { + @Override + protected void afterExecute(Runnable task, Throwable failure) { + afterExecute.countDown(); + try { + assertThat(releaseWorker.await(5, SECONDS)).isTrue(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + } + }; + } + + private static void prepare(InvocationOnMock invocation) { + RetainedCutoverPreparation preparation = invocation.getArgument(5); + preparation.prepare(new RetainedCutoverPreparationContext(OPERATION, IDENTITY), + invocation.getArgument(1), invocation.getArgument(2)); + } + + private static RetainedCutoverResult retained() { + return new RetainedCutoverResult( + OPERATION, IDENTITY, RetainedCutoverResult.Status.RETAINED_SUCCESS); + } + + private ManagedMigrationConfigurationTransaction configuration() { + ManagedMigrationConfigurationTransaction configuration = mock( + ManagedMigrationConfigurationTransaction.class); + try { + when(configuration.stageMetadataTarget(any(), any(), any(), any(), any())) + .thenAnswer(DeploymentMigrationCommandRunnerReviewTest::stage); + } catch (IOException impossible) { + throw new AssertionError(impossible); + } + return configuration; + } + + private static MetadataTargetStageResult stage(InvocationOnMock invocation) { + return new MetadataTargetStageResult(StageOutcome.STAGED, + Optional.of(new CandidateRef(invocation.getArgument(0), invocation.getArgument(1)))); + } + + private static MetadataMigrationRequest request() { + return new MetadataMigrationRequest( + OPERATION, MigrationTarget.MYSQL, + new MetadataDatabaseConfiguration(MetadataDatabaseKind.MYSQL, + "jdbc:mysql://db.example/hertzbeat", "migration", "password-a"), + ApplyMode.MANAGED_WRITE); + } + + private static void awaitPhase( + DeploymentMigrationCommandRunner runner, RetainedCutoverRecoveryPhase phase) { + long deadline = System.nanoTime() + SECONDS.toNanos(5); + while (!runner.activeRecoveryPhase().equals(Optional.of(phase)) + && System.nanoTime() < deadline) { + Thread.onSpinWait(); + } + assertThat(runner.activeRecoveryPhase()).contains(phase); + } + + private static void awaitTerminal(FileMigrationOperationStore store) { + long deadline = System.nanoTime() + SECONDS.toNanos(5); + while (store.find(OPERATION).filter(MigrationOperationSnapshot::terminal).isEmpty() + && System.nanoTime() < deadline) { + Thread.onSpinWait(); + } + assertThat(store.find(OPERATION)).hasValueSatisfying( + snapshot -> assertThat(snapshot.terminal()).isTrue()); + } + + private static void awaitState(Thread thread, Thread.State expected) { + long deadline = System.nanoTime() + SECONDS.toNanos(5); + while (thread.getState() != expected && System.nanoTime() < deadline) { + Thread.onSpinWait(); + } + assertThat(thread.getState()).isEqualTo(expected); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationCommandRunnerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationCommandRunnerTest.java new file mode 100644 index 0000000000..16aa0c8264 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationCommandRunnerTest.java @@ -0,0 +1,451 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.HexFormat; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MetadataMigrationRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationView; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.CandidateRef; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.MetadataTargetStageResult; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.StageOutcome; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; + +@Timeout(15) +class DeploymentMigrationCommandRunnerTest { + + private static final String OPERATION = "operation-a"; + private static final String IDENTITY = "a".repeat(64); + private static final Instant NOW = Instant.parse("2026-08-10T04:00:00Z"); + + @TempDir + private Path root; + + @Test + void returnsOnlyAfterDurableRunningBarrierAndBeforeCopyCompletion() throws Exception { + Fixture fixture = fixture(); + CountDownLatch preparationReturned = new CountDownLatch(1); + CountDownLatch releaseCopy = new CountDownLatch(1); + when(fixture.coordinator.execute(any(), any(), any(), any(), any(), any(), any())) + .thenAnswer(invocation -> { + RetainedCutoverPreparation preparation = invocation.getArgument(5); + preparation.prepare( + new RetainedCutoverPreparationContext(OPERATION, IDENTITY), + invocation.getArgument(1), invocation.getArgument(2)); + preparationReturned.countDown(); + assertThat(releaseCopy.await(5, SECONDS)).isTrue(); + return new RetainedCutoverResult( + OPERATION, IDENTITY, RetainedCutoverResult.Status.RETAINED_SUCCESS); + }); + + try (DeploymentMigrationCommandRunner runner = fixture.runner()) { + try { + MigrationView view = runner.start(request(OPERATION, ApplyMode.MANAGED_WRITE, "password-a")); + + assertThat(preparationReturned.getCount()).isZero(); + assertThat(view.state()).isEqualTo(MigrationOperationState.RUNNING); + assertThat(view.stage()).isEqualTo(MigrationStage.COPYING); + assertThat(view.progressPercent()).isZero(); + assertThat(fixture.store.find(OPERATION).orElseThrow().managedCandidateGeneration()) + .isEqualTo(generation(OPERATION)); + } finally { + releaseCopy.countDown(); + } + } + } + + @Test + void sameActiveOperationJoinsOneBarrierWithoutDereferencingReplacementPassword() throws Exception { + Fixture fixture = fixture(); + CountDownLatch releaseCopy = new CountDownLatch(1); + when(fixture.coordinator.execute(any(), any(), any(), any(), any(), any(), any())) + .thenAnswer(invocation -> { + RetainedCutoverPreparation preparation = invocation.getArgument(5); + preparation.prepare( + new RetainedCutoverPreparationContext(OPERATION, IDENTITY), + invocation.getArgument(1), invocation.getArgument(2)); + assertThat(releaseCopy.await(5, SECONDS)).isTrue(); + return new RetainedCutoverResult( + OPERATION, IDENTITY, RetainedCutoverResult.Status.RETAINED_SUCCESS); + }); + + try (DeploymentMigrationCommandRunner runner = fixture.runner()) { + try { + MigrationView first = runner.start(request(OPERATION, ApplyMode.MANAGED_WRITE, "password-a")); + MigrationView joined = runner.start(request(OPERATION, ApplyMode.MANAGED_WRITE, null)); + + assertThat(joined).isEqualTo(first); + assertStoreError(SetupErrorCode.OPERATION_CONFLICT, + () -> runner.start(mysqlRequest( + OPERATION, "jdbc:mysql://other.example/hertzbeat", "migration", null))); + verify(fixture.coordinator).execute(any(), any(), any(), any(), any(), any(), any()); + } finally { + releaseCopy.countDown(); + } + } + } + + @Test + void persistedRunningAndReadyReplayWithoutCredentialOrCoordinatorAccess() { + assertPersistedReplay(running(27)); + assertPersistedReplay(ready()); + } + + @Test + void externalApplyIsRejectedBeforeSlotSecretAndCorruptStoreAccess() throws Exception { + Path journal = root.resolve(FileMigrationOperationStore.RELATIVE_PATH); + java.nio.file.Files.createDirectories(journal.getParent()); + java.nio.file.Files.writeString(journal, "schema=99\n"); + Fixture fixture = fixture(); + + try (DeploymentMigrationCommandRunner runner = fixture.runner()) { + assertStoreError(SetupErrorCode.INVALID_REQUEST, () -> runner.start( + request(OPERATION, ApplyMode.EXTERNAL_APPLY, null))); + } + verify(fixture.coordinator, never()).execute(any(), any(), any(), any(), any(), any(), any()); + } + + @Test + void foreignActiveAndPersistedTargetOrApplyMismatchConflict() { + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + store.create(pending("operation-b", MigrationTarget.MYSQL, ApplyMode.MANAGED_WRITE)); + Fixture foreign = fixture(store); + try (DeploymentMigrationCommandRunner runner = foreign.runner()) { + assertStoreError(SetupErrorCode.OPERATION_CONFLICT, + () -> runner.start(request(OPERATION, ApplyMode.MANAGED_WRITE, "password-a"))); + } + + Path secondRoot = root.resolve("second"); + FileMigrationOperationStore mismatchStore = new FileMigrationOperationStore(secondRoot); + mismatchStore.create(pending(OPERATION, MigrationTarget.MYSQL, ApplyMode.MANAGED_WRITE)); + mismatchStore.compareAndTransition( + OPERATION, MigrationOperationState.PENDING, running(0)); + Fixture mismatch = fixture(mismatchStore); + try (DeploymentMigrationCommandRunner runner = mismatch.runner()) { + assertStoreError(SetupErrorCode.OPERATION_CONFLICT, + () -> runner.start(postgresRequest(OPERATION))); + } + + FileMigrationOperationStore applyStore = new FileMigrationOperationStore(root.resolve("apply")); + applyStore.create(pending(OPERATION, MigrationTarget.MYSQL, ApplyMode.EXTERNAL_APPLY)); + Fixture applyMismatch = fixture(applyStore); + try (DeploymentMigrationCommandRunner runner = applyMismatch.runner()) { + assertStoreError(SetupErrorCode.OPERATION_CONFLICT, + () -> runner.start(request(OPERATION, ApplyMode.MANAGED_WRITE, "password-a"))); + } + } + + @Test + void preparationErrorStillPublishesAuthoritativeTerminalViewBeforeWorkerPropagates() throws Exception { + Fixture fixture = fixture(); + doReturn(new MetadataTargetStageResult(StageOutcome.SOURCE_UNSUPPORTED, Optional.empty())) + .when(fixture.configuration).stageMetadataTarget(any(), any(), any(), any(), any()); + when(fixture.coordinator.execute(any(), any(), any(), any(), any(), any(), any())) + .thenAnswer(invocation -> { + RetainedCutoverPreparation preparation = invocation.getArgument(5); + preparation.prepare( + new RetainedCutoverPreparationContext(OPERATION, IDENTITY), + invocation.getArgument(1), invocation.getArgument(2)); + throw new AssertionError("unreachable"); + }); + + try (DeploymentMigrationCommandRunner runner = fixture.runner()) { + MigrationView view = runner.start(request(OPERATION, ApplyMode.MANAGED_WRITE, "password-a")); + assertThat(view.state()).isEqualTo(MigrationOperationState.FAILED); + assertThat(view.errorCode()).isEqualTo(SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED); + } + } + + @Test + void preparationErrorWinsAfterAuthoritativeJournalValidation() throws Exception { + Fixture fixture = fixture(); + AssertionError fatal = new AssertionError("fatal-preparation"); + doThrow(fatal).when(fixture.configuration) + .stageMetadataTarget(any(), any(), any(), any(), any()); + when(fixture.coordinator.execute(any(), any(), any(), any(), any(), any(), any())) + .thenAnswer(invocation -> { + RetainedCutoverPreparation preparation = invocation.getArgument(5); + preparation.prepare( + new RetainedCutoverPreparationContext(OPERATION, IDENTITY), + invocation.getArgument(1), invocation.getArgument(2)); + throw new AssertionError("unreachable"); + }); + + try (DeploymentMigrationCommandRunner runner = fixture.runner()) { + assertThatThrownBy(() -> runner.start( + request(OPERATION, ApplyMode.MANAGED_WRITE, "password-a"))) + .isSameAs(fatal); + } + assertThat(fixture.store.find(OPERATION).orElseThrow().state()) + .isEqualTo(MigrationOperationState.PENDING); + } + + @Test + void workerStartTimeDoesNotReusePersistedPendingCreationTime() throws Exception { + Instant createdAt = NOW.minusSeconds(300); + FileMigrationOperationStore store = new FileMigrationOperationStore(root.resolve("started-at")); + store.create(pendingAt(createdAt)); + Fixture fixture = fixture(store); + when(fixture.coordinator.execute(any(), any(), any(), any(), any(), any(), any())) + .thenAnswer(invocation -> { + RetainedCutoverPreparation preparation = invocation.getArgument(5); + preparation.prepare( + new RetainedCutoverPreparationContext(OPERATION, IDENTITY), + invocation.getArgument(1), invocation.getArgument(2)); + return new RetainedCutoverResult( + OPERATION, IDENTITY, RetainedCutoverResult.Status.RETAINED_SUCCESS); + }); + AtomicReference currentTime = new AtomicReference<>(NOW); + Clock clock = mock(Clock.class); + when(clock.instant()).thenAnswer(ignored -> currentTime.get()); + ExecutorService worker = Executors.newSingleThreadExecutor(); + CountDownLatch workerOccupied = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + worker.execute(() -> { + workerOccupied.countDown(); + try { + assertThat(releaseWorker.await(5, SECONDS)).isTrue(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + }); + assertThat(workerOccupied.await(5, SECONDS)).isTrue(); + DeploymentMigrationCommandRunner runner = new DeploymentMigrationCommandRunner( + store, fixture.configuration, fixture.coordinator, clock, + Duration.ofSeconds(30), worker); + CompletableFuture started = CompletableFuture.supplyAsync(() -> runner.start( + request(OPERATION, ApplyMode.MANAGED_WRITE, "password-a"))); + try { + awaitActive(runner); + Instant workerStartedAt = NOW.plusSeconds(45); + currentTime.set(workerStartedAt); + releaseWorker.countDown(); + + assertThat(started.get(5, SECONDS).startedAt()).isEqualTo(workerStartedAt); + assertThat(store.find(OPERATION).orElseThrow().startedAt()).isEqualTo(workerStartedAt); + } finally { + releaseWorker.countDown(); + runner.close(); + } + } + + @Test + void taskOwnsCopiedSecretAndNeverRetainsTheRequestDto() throws Exception { + Fixture fixture = fixture(); + AtomicReference captured = new AtomicReference<>(); + CountDownLatch workerExited = new CountDownLatch(1); + when(fixture.coordinator.execute(any(), any(), any(), any(), any(), any(), any())) + .thenAnswer(invocation -> { + captured.set(invocation.getArgument(2)); + RetainedCutoverPreparation preparation = invocation.getArgument(5); + preparation.prepare( + new RetainedCutoverPreparationContext(OPERATION, IDENTITY), + invocation.getArgument(1), invocation.getArgument(2)); + workerExited.countDown(); + throw new AssertionError("fatal-after-preparation"); + }); + + try (DeploymentMigrationCommandRunner runner = fixture.runner()) { + runner.start(request(OPERATION, ApplyMode.MANAGED_WRITE, "password-a")); + assertThat(workerExited.await(5, SECONDS)).isTrue(); + } + assertThat(captured.get().copy()).containsOnly('\0'); + assertThat(MigrationCommandTask.class.getDeclaredFields()) + .allSatisfy(field -> assertThat(field.getType()).isNotEqualTo(MetadataMigrationRequest.class)); + } + + private void assertPersistedReplay(MigrationOperationSnapshot snapshot) { + Path replayRoot = root.resolve(snapshot.stage().name()); + FileMigrationOperationStore store = new FileMigrationOperationStore(replayRoot); + store.create(pending(OPERATION, MigrationTarget.MYSQL, ApplyMode.MANAGED_WRITE)); + store.compareAndTransition(OPERATION, MigrationOperationState.PENDING, running(0)); + if (!snapshot.equals(running(0))) { + if (snapshot.stage() == MigrationStage.COPYING) { + store.compareAndTransition(OPERATION, MigrationOperationState.RUNNING, snapshot); + } else { + store.compareAndTransition(OPERATION, MigrationOperationState.RUNNING, verifying()); + store.compareAndTransition(OPERATION, MigrationOperationState.RUNNING, snapshot); + } + } + Fixture fixture = fixture(store); + try (DeploymentMigrationCommandRunner runner = fixture.runner()) { + MigrationView view = runner.start(request(OPERATION, ApplyMode.MANAGED_WRITE, null)); + assertThat(view.state()).isEqualTo(snapshot.state()); + assertThat(view.stage()).isEqualTo(snapshot.stage()); + } + verify(fixture.coordinator, never()).execute(any(), any(), any(), any(), any(), any(), any()); + } + + private Fixture fixture() { + return fixture(new FileMigrationOperationStore(root)); + } + + private Fixture fixture(FileMigrationOperationStore store) { + RetainedCutoverCoordinator coordinator = mock(RetainedCutoverCoordinator.class); + ManagedMigrationConfigurationTransaction configuration = mock( + ManagedMigrationConfigurationTransaction.class); + try { + when(configuration.stageMetadataTarget(any(), any(), any(), any(), any())) + .thenAnswer(invocation -> new MetadataTargetStageResult( + StageOutcome.STAGED, + Optional.of(new CandidateRef( + invocation.getArgument(0), invocation.getArgument(1))))); + } catch (java.io.IOException impossible) { + throw new AssertionError(impossible); + } + return new Fixture(store, coordinator, configuration); + } + + private record Fixture( + FileMigrationOperationStore store, + RetainedCutoverCoordinator coordinator, + ManagedMigrationConfigurationTransaction configuration) { + + DeploymentMigrationCommandRunner runner() { + ExecutorService worker = Executors.newSingleThreadExecutor(); + return new DeploymentMigrationCommandRunner( + store, configuration, coordinator, Clock.fixed(NOW, ZoneOffset.UTC), + Duration.ofSeconds(30), worker); + } + } + + private static MetadataMigrationRequest request( + String operationId, ApplyMode applyMode, String password) { + return mysqlRequest( + operationId, "jdbc:mysql://db.example/hertzbeat", "migration", password, applyMode); + } + + private static MetadataMigrationRequest mysqlRequest( + String operationId, String jdbcUrl, String username, String password) { + return mysqlRequest(operationId, jdbcUrl, username, password, ApplyMode.MANAGED_WRITE); + } + + private static MetadataMigrationRequest mysqlRequest( + String operationId, String jdbcUrl, String username, String password, ApplyMode applyMode) { + return new MetadataMigrationRequest( + operationId, MigrationTarget.MYSQL, + new MetadataDatabaseConfiguration( + MetadataDatabaseKind.MYSQL, jdbcUrl, username, password), + applyMode); + } + + private static MetadataMigrationRequest postgresRequest(String operationId) { + return new MetadataMigrationRequest( + operationId, MigrationTarget.POSTGRESQL, + new MetadataDatabaseConfiguration( + MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://db.example/hertzbeat", "migration", null), + ApplyMode.MANAGED_WRITE); + } + + private static MigrationOperationSnapshot pending( + String operationId, MigrationTarget target, ApplyMode applyMode) { + return snapshot(operationId, target, applyMode, MigrationOperationState.PENDING, + MigrationStage.QUEUED, 0, null, VerificationState.PENDING, null, 1000); + } + + private static MigrationOperationSnapshot pendingAt(Instant createdAt) { + return new MigrationOperationSnapshot( + OPERATION, MigrationOperationState.PENDING, MigrationTarget.MYSQL, + ApplyMode.MANAGED_WRITE, MigrationStage.QUEUED, 0, createdAt, + null, null, VerificationState.PENDING, null, null, 1000, + false, false, false, IDENTITY, generation(OPERATION)); + } + + private static MigrationOperationSnapshot running(int progress) { + return snapshot(OPERATION, MigrationTarget.MYSQL, ApplyMode.MANAGED_WRITE, + MigrationOperationState.RUNNING, MigrationStage.COPYING, progress, NOW, + VerificationState.PENDING, null, 1000); + } + + private static MigrationOperationSnapshot verifying() { + return snapshot(OPERATION, MigrationTarget.MYSQL, ApplyMode.MANAGED_WRITE, + MigrationOperationState.RUNNING, MigrationStage.VERIFYING, 100, NOW, + VerificationState.RUNNING, null, 1000); + } + + private static MigrationOperationSnapshot ready() { + return snapshot(OPERATION, MigrationTarget.MYSQL, ApplyMode.MANAGED_WRITE, + MigrationOperationState.READY_TO_ACTIVATE, MigrationStage.READY_TO_ACTIVATE, 100, NOW, + VerificationState.SUCCEEDED, null, 0); + } + + private static MigrationOperationSnapshot snapshot( + String operationId, MigrationTarget target, ApplyMode applyMode, + MigrationOperationState state, MigrationStage stage, int progress, + Instant startedAt, VerificationState verification, SetupErrorCode error, long poll) { + return new MigrationOperationSnapshot( + operationId, state, target, applyMode, stage, progress, NOW, startedAt, null, + verification, error, null, poll, + state == MigrationOperationState.READY_TO_ACTIVATE, false, false, + IDENTITY, applyMode == ApplyMode.MANAGED_WRITE ? generation(operationId) : null); + } + + private static String generation(String operationId) { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(operationId.getBytes(StandardCharsets.UTF_8))); + } catch (java.security.NoSuchAlgorithmException impossible) { + throw new AssertionError(impossible); + } + } + + private static void assertStoreError(SetupErrorCode code, ThrowingAction action) { + assertThatThrownBy(action::run) + .isInstanceOfSatisfying(MigrationOperationStoreException.class, + failure -> assertThat(failure.errorCode()).isEqualTo(code)); + } + + private static void awaitActive(DeploymentMigrationCommandRunner runner) { + long deadline = System.nanoTime() + SECONDS.toNanos(5); + while (runner.activeOperationId().isEmpty() && System.nanoTime() < deadline) { + Thread.onSpinWait(); + } + assertThat(runner.activeOperationId()).contains(OPERATION); + } + + @FunctionalInterface + private interface ThrowingAction { + void run(); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverFailureFinalizerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverFailureFinalizerTest.java index 791d591160..c3b2629e8e 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverFailureFinalizerTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverFailureFinalizerTest.java @@ -86,7 +86,10 @@ class DurableCutoverFailureFinalizerTest { @Test void acceptsOnlyTheClosedKnownCleanupFailureCapability() { assertThat(DurableKnownFailure.values()) - .containsExactly(DurableKnownFailure.COPY, DurableKnownFailure.VERIFICATION); + .containsExactly( + DurableKnownFailure.COPY, + DurableKnownFailure.VERIFICATION, + DurableKnownFailure.CURRENT_PHASE); assertThat(DurableCutoverFailureFinalizer.class.getDeclaredMethods()) .filteredOn(method -> method.getName().equals("finalizeFailure")) .singleElement() diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStoreExactTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStoreExactTest.java index 28e8c1c208..3f2fc56c1f 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStoreExactTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStoreExactTest.java @@ -163,6 +163,27 @@ class FileMigrationOperationStoreExactTest { assertThat(publications).hasValue(5); } + @Test + void exactSnapshotReplayMustRepublishBeforeReturningSuccess() { + MigrationOperationSnapshot pending = pending("operation-a", IDENTITY, GENERATION); + new FileMigrationOperationStore(root).create(pending); + MigrationOperationFilePublisher committed = new MigrationOperationFilePublisher(root); + AtomicInteger publications = new AtomicInteger(); + FileMigrationOperationStore uncertain = new FileMigrationOperationStore(root, (target, content) -> { + committed.publish(target, content); + if (publications.incrementAndGet() <= 4) { + throw new CommittedSetupFileDurabilityException(); + } + }); + + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> uncertain.confirmExactForStartup(pending)); + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> uncertain.confirmExactForStartup(pending)); + assertThat(uncertain.confirmExactForStartup(pending)).isEqualTo(pending); + assertThat(publications).hasValue(5); + } + @Test void uncertainCreateMissingOrCorruptFailsClosed() { MigrationOperationSnapshot pending = pending("operation-a", IDENTITY, GENERATION); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationPreparationBarrierTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationPreparationBarrierTest.java new file mode 100644 index 0000000000..c5354473dc --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationPreparationBarrierTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.time.Duration; +import java.time.Instant; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.junit.jupiter.api.Test; + +class MigrationPreparationBarrierTest { + + private static final String OPERATION = "operation-a"; + private static final String IDENTITY = "a".repeat(64); + + @Test + void firstPreparationErrorWinsWhenAuthoritativeReadAlsoThrowsError() { + FileMigrationOperationStore store = mock(FileMigrationOperationStore.class); + AssertionError journalFatal = new AssertionError("private-journal-failure"); + when(store.selectForStartup(OPERATION)).thenThrow(journalFatal); + AssertionError preparationFatal = new AssertionError("fatal-preparation"); + RetainedCutoverPreparation delegate = (context, target, password) -> { + throw preparationFatal; + }; + MigrationPreparationBarrier barrier = new MigrationPreparationBarrier(store); + barrier.bind(draft(), delegate); + + try (SecretValue password = SecretValue.of("password-a")) { + assertThatThrownBy(() -> barrier.prepare( + new RetainedCutoverPreparationContext(OPERATION, IDENTITY), + new MetadataDatabaseSettings( + MetadataDatabaseKind.MYSQL, + "jdbc:mysql://db.example/hertzbeat", "migration"), + password)).isSameAs(preparationFatal); + } + assertThatThrownBy(() -> barrier.await(Duration.ofSeconds(1))).isSameAs(preparationFatal); + assertThat(preparationFatal.getSuppressed()).singleElement() + .isInstanceOfSatisfying(MigrationOperationStoreException.class, failure -> { + assertThat(failure.errorCode()).isEqualTo(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + assertThat(failure).hasNoCause(); + assertThat(failure.getMessage()).doesNotContain("journal"); + }); + assertThat(preparationFatal.getSuppressed()).doesNotContain(journalFatal); + } + + private static DurableCutoverDraft draft() { + Instant now = Instant.parse("2026-08-10T06:00:00Z"); + return new DurableCutoverDraft( + OPERATION, MigrationTarget.MYSQL, ApplyMode.MANAGED_WRITE, + now, now, "b".repeat(64)); + } +} From 420d280e5d77ce797413c1abfa58cfa8f80966da Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 21:34:04 +0800 Subject: [PATCH 67/71] Manage retained migration lifecycle --- .../workflow/DurableCutoverDraftFactory.java | 31 ++ .../ManagedDeploymentMigrationCommands.java | 168 +++++++++ .../workflow/RetainedCutoverCoordinator.java | 9 + .../workflow/RetainedCutoverShutdown.java | 41 +++ .../setup/workflow/RetainedCutoverState.java | 18 + .../setup/workflow/RetainedCutoverStatus.java | 47 +++ .../DurableCutoverDraftFactoryTest.java | 67 ++++ ...anagedDeploymentMigrationCommandsTest.java | 332 ++++++++++++++++++ .../workflow/RetainedCutoverShutdownTest.java | 191 ++++++++++ .../workflow/RetainedCutoverStatusTest.java | 119 +++++++ 10 files changed, 1023 insertions(+) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverDraftFactory.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedDeploymentMigrationCommands.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverShutdown.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverStatus.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverDraftFactoryTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedDeploymentMigrationCommandsTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverShutdownTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverStatusTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverDraftFactory.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverDraftFactory.java new file mode 100644 index 0000000000..3232af326c --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverDraftFactory.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; + +/** Reconstructs a secret-free cutover identity only from a complete validated journal record. */ +final class DurableCutoverDraftFactory { + + private DurableCutoverDraftFactory() { + } + + static DurableCutoverDraft from(MigrationOperationSnapshot snapshot) { + Objects.requireNonNull(snapshot, "snapshot"); + if (snapshot.applyMode() != ApplyMode.MANAGED_WRITE + || snapshot.startedAt() == null + || snapshot.managedCandidateGeneration() == null) { + throw new MigrationOperationStoreException(SetupErrorCode.OPERATION_CONFLICT); + } + return new DurableCutoverDraft( + snapshot.operationId(), snapshot.target(), snapshot.applyMode(), snapshot.createdAt(), + snapshot.startedAt(), snapshot.managedCandidateGeneration()); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedDeploymentMigrationCommands.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedDeploymentMigrationCommands.java new file mode 100644 index 0000000000..5eb16a073f --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedDeploymentMigrationCommands.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.ActivateMigrationRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MetadataMigrationRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationView; +import org.apache.hertzbeat.manager.setup.api.OperationIdValidator; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction; + +/** Spring-free managed migration command facade over one exact retained-cutover graph. */ +final class ManagedDeploymentMigrationCommands implements AutoCloseable { + + private final DeploymentMigrationCommandRunner runner; + private final FileMigrationOperationStore store; + private final ManagedMigrationConfigurationTransaction configuration; + private final RetainedCutoverCoordinator coordinator; + private volatile boolean closed; + + ManagedDeploymentMigrationCommands( + DeploymentMigrationCommandRunner runner, + FileMigrationOperationStore store, + ManagedMigrationConfigurationTransaction configuration, + RetainedCutoverCoordinator coordinator) { + this.runner = Objects.requireNonNull(runner, "runner"); + this.store = Objects.requireNonNull(store, "store"); + this.configuration = Objects.requireNonNull(configuration, "configuration"); + this.coordinator = Objects.requireNonNull(coordinator, "coordinator"); + } + + MigrationView migrate(MetadataMigrationRequest request) { + requireOpen(); + Objects.requireNonNull(request, "request"); + if (request.applyMode() != ApplyMode.MANAGED_WRITE) { + throw failure(SetupErrorCode.INVALID_REQUEST); + } + return runner.start(request); + } + + MigrationView migration(String operationId) { + requireOpen(); + requireOperationId(operationId); + Optional stored = runner.find(operationId); + RetainedCutoverStatus retained = coordinator.status(); + if (retained.owns(operationId) && !matchesRetainedShape(stored, retained.phase())) { + throw failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + return stored.orElseThrow(() -> failure(SetupErrorCode.OPERATION_NOT_FOUND)); + } + + MigrationView activate(String operationId, ActivateMigrationRequest request) { + requireOpen(); + requireOperationId(operationId); + Objects.requireNonNull(request, "request"); + if (request.expectedState() != MigrationOperationState.READY_TO_ACTIVATE) { + throw failure(SetupErrorCode.OPERATION_CONFLICT); + } + Optional stored = store.find(operationId); + RetainedCutoverStatus retained = coordinator.status(); + if (stored.isEmpty()) { + if (retained.owns(operationId)) { + throw failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + throw failure(SetupErrorCode.OPERATION_NOT_FOUND); + } + requireOwned(retained, operationId); + MigrationOperationSnapshot current = stored.orElseThrow( + () -> failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)); + activate(current, retained.phase()); + return confirmedAwaitingRestart(operationId); + } + + Optional activeOperationId() { + requireOpen(); + Optional durable = runner.activeOperationId(); + RetainedCutoverStatus retained = coordinator.status(); + if (retained.phase() == RetainedCutoverStatus.Phase.NONE) { + return durable; + } + if (durable.isPresent() && !durable.get().equals(retained.operationId())) { + throw failure(SetupErrorCode.OPERATION_CONFLICT); + } + return Optional.of(retained.operationId()); + } + + @Override + public synchronized void close() { + closed = true; + runner.close(); + RetainedCutoverStatus retained = coordinator.status(); + if (retained.phase() != RetainedCutoverStatus.Phase.NONE) { + coordinator.shutdownOperation(retained.operationId()); + } + } + + private void activate( + MigrationOperationSnapshot current, RetainedCutoverStatus.Phase phase) { + switch (phase) { + case RETAINED -> activateReady(current); + case ACTIVATION_PENDING, AWAITING_RESTART_RETAINED -> + coordinator.retryActivation(current.operationId()); + default -> throw failure(SetupErrorCode.OPERATION_CONFLICT); + } + } + + private void activateReady(MigrationOperationSnapshot current) { + if (current.state() != MigrationOperationState.READY_TO_ACTIVATE) { + throw failure(SetupErrorCode.MIGRATION_ACTIVATION_NOT_AVAILABLE); + } + DurableCutoverDraft draft = DurableCutoverDraftFactory.from(current); + coordinator.activateRetained(current.operationId(), + new DurableRetainedManagedActivation(draft, store, configuration)); + } + + private MigrationView confirmedAwaitingRestart(String operationId) { + MigrationOperationSnapshot current = store.find(operationId) + .orElseThrow(() -> failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)); + RetainedCutoverStatus retained = coordinator.status(); + if (!retained.owns(operationId) + || retained.phase() != RetainedCutoverStatus.Phase.AWAITING_RESTART_RETAINED + || current.state() != MigrationOperationState.AWAITING_RESTART) { + throw failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + return MigrationOperationProjection.view(current); + } + + private static boolean matchesRetainedShape( + Optional stored, RetainedCutoverStatus.Phase phase) { + return stored.filter(view -> switch (phase) { + case RETAINED -> view.state() == MigrationOperationState.READY_TO_ACTIVATE; + case AWAITING_RESTART_RETAINED -> + view.state() == MigrationOperationState.AWAITING_RESTART; + default -> false; + }).isPresent(); + } + + private static void requireOwned(RetainedCutoverStatus retained, String operationId) { + if (!retained.owns(operationId)) { + throw failure(SetupErrorCode.OPERATION_CONFLICT); + } + } + + private void requireOpen() { + if (closed) { + throw failure(SetupErrorCode.MIGRATION_UNAVAILABLE); + } + } + + private static void requireOperationId(String operationId) { + if (!OperationIdValidator.isSafe(operationId)) { + throw failure(SetupErrorCode.INVALID_REQUEST); + } + } + + private static MigrationOperationStoreException failure(SetupErrorCode code) { + return new MigrationOperationStoreException(code); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java index d7fc29eb36..fc7611fb60 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverCoordinator.java @@ -100,6 +100,15 @@ final class RetainedCutoverCoordinator { return state.retained(operationId); } + RetainedCutoverStatus status() { + return state.status(); + } + + void shutdownOperation(String operationId) { + requireOperationId(operationId); + RetainedCutoverShutdown.run(this, operationId); + } + void releaseRetained(String operationId) { requireOperationId(operationId); RetainedCutoverState.Execution execution = state.claimRetainedRelease(operationId); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverShutdown.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverShutdown.java new file mode 100644 index 0000000000..4094a7e377 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverShutdown.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.time.Duration; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceException; + +/** Advances only exact retryable cleanup phases during coordinator shutdown. */ +final class RetainedCutoverShutdown { + + private static final Duration CLEANUP_TIMEOUT = Duration.ofSeconds(30); + + private RetainedCutoverShutdown() { } + + static void run(RetainedCutoverCoordinator coordinator, String operationId) { + while (true) { + RetainedCutoverStatus status = coordinator.status(); + if (status.phase() == RetainedCutoverStatus.Phase.NONE) { + return; + } + if (!status.owns(operationId)) { + throw MigrationMaintenanceException.operationConflict(); + } + if (status.phase() == RetainedCutoverStatus.Phase.AWAITING_RESTART_RETAINED) { + return; + } + switch (status.phase()) { + case HANDOFF_PENDING -> coordinator.retryHandoff(operationId); + case RETAINED -> coordinator.releaseRetained(operationId); + case ACTIVATION_PENDING -> coordinator.retryActivation(operationId); + case RELEASE_PENDING -> coordinator.retryRelease(operationId, CLEANUP_TIMEOUT); + default -> throw MigrationMaintenanceException.operationConflict(); + } + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverState.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverState.java index 19fca811b5..3a57449e8e 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverState.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverState.java @@ -16,6 +16,24 @@ final class RetainedCutoverState { private Execution active; + synchronized RetainedCutoverStatus status() { + if (active == null) { + return RetainedCutoverStatus.empty(); + } + return new RetainedCutoverStatus(active.operationId, switch (active.phase) { + case EXECUTING -> RetainedCutoverStatus.Phase.EXECUTING; + case HANDOFFING -> RetainedCutoverStatus.Phase.HANDOFFING; + case HANDOFF_PENDING -> RetainedCutoverStatus.Phase.HANDOFF_PENDING; + case RETAINED -> RetainedCutoverStatus.Phase.RETAINED; + case ACTIVATING -> RetainedCutoverStatus.Phase.ACTIVATING; + case ACTIVATION_PENDING -> RetainedCutoverStatus.Phase.ACTIVATION_PENDING; + case AWAITING_RESTART_RETAINED -> + RetainedCutoverStatus.Phase.AWAITING_RESTART_RETAINED; + case RELEASING -> RetainedCutoverStatus.Phase.RELEASING; + case RELEASE_PENDING -> RetainedCutoverStatus.Phase.RELEASE_PENDING; + }); + } + synchronized Execution reserve(String operationId, RetainedCopyJournalHandoff handoff) { if (active != null) { throw MigrationMaintenanceException.operationConflict(); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverStatus.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverStatus.java new file mode 100644 index 0000000000..9bfd8399f4 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverStatus.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.OperationIdValidator; + +/** Secret-free total projection of the process-local retained-cutover capability. */ +record RetainedCutoverStatus(String operationId, Phase phase) { + + RetainedCutoverStatus { + Objects.requireNonNull(phase, "phase"); + if (phase == Phase.NONE) { + if (operationId != null) { + throw new IllegalArgumentException("Empty retained status cannot own an operation"); + } + } else if (!OperationIdValidator.isSafe(operationId)) { + throw new IllegalArgumentException("Invalid retained operation identifier"); + } + } + + static RetainedCutoverStatus empty() { + return new RetainedCutoverStatus(null, Phase.NONE); + } + + boolean owns(String requestedOperationId) { + return operationId != null && operationId.equals(requestedOperationId); + } + + enum Phase { + NONE, + EXECUTING, + HANDOFFING, + HANDOFF_PENDING, + RETAINED, + ACTIVATING, + ACTIVATION_PENDING, + AWAITING_RESTART_RETAINED, + RELEASING, + RELEASE_PENDING + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverDraftFactoryTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverDraftFactoryTest.java new file mode 100644 index 0000000000..cfea6efe85 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DurableCutoverDraftFactoryTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.time.Instant; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.junit.jupiter.api.Test; + +class DurableCutoverDraftFactoryTest { + + private static final Instant CREATED = Instant.parse("2026-08-10T01:00:00Z"); + private static final Instant STARTED = CREATED.plusSeconds(1); + + @Test + void derivesEveryImmutableFieldFromTheValidatedJournalSnapshot() { + MigrationOperationSnapshot snapshot = snapshot(ApplyMode.MANAGED_WRITE, STARTED, "generation-a"); + + DurableCutoverDraft draft = DurableCutoverDraftFactory.from(snapshot); + + assertThat(draft.operationId()).isEqualTo("operation-a"); + assertThat(draft.target()).isEqualTo(MigrationTarget.POSTGRESQL); + assertThat(draft.applyMode()).isEqualTo(ApplyMode.MANAGED_WRITE); + assertThat(draft.createdAt()).isEqualTo(CREATED); + assertThat(draft.startedAt()).isEqualTo(STARTED); + assertThat(draft.candidateGeneration()).isEqualTo("generation-a"); + } + + @Test + void rejectsPrePreparationAndExternalSnapshotsWithoutInventingIdentity() { + assertThatThrownBy(() -> DurableCutoverDraftFactory.from(pending())) + .isInstanceOf(MigrationOperationStoreException.class) + .hasNoCause(); + assertThatThrownBy(() -> DurableCutoverDraftFactory.from( + snapshot(ApplyMode.EXTERNAL_APPLY, STARTED, null))) + .isInstanceOf(MigrationOperationStoreException.class) + .hasNoCause(); + } + + private static MigrationOperationSnapshot snapshot( + ApplyMode applyMode, Instant startedAt, String generation) { + return new MigrationOperationSnapshot( + "operation-a", MigrationOperationState.RUNNING, MigrationTarget.POSTGRESQL, + applyMode, MigrationStage.ACTIVATING, 100, CREATED, startedAt, null, + VerificationState.SUCCEEDED, null, null, 250, false, false, false, + "a".repeat(64), generation); + } + + private static MigrationOperationSnapshot pending() { + return new MigrationOperationSnapshot( + "operation-a", MigrationOperationState.PENDING, MigrationTarget.POSTGRESQL, + ApplyMode.MANAGED_WRITE, MigrationStage.QUEUED, 0, CREATED, null, null, + VerificationState.PENDING, null, null, 250, false, false, false, + "a".repeat(64), "generation-a"); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedDeploymentMigrationCommandsTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedDeploymentMigrationCommandsTest.java new file mode 100644 index 0000000000..29fcacdb66 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedDeploymentMigrationCommandsTest.java @@ -0,0 +1,332 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.nio.file.Path; +import java.time.Instant; +import java.util.Optional; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.ActivateMigrationRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MetadataMigrationRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationView; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.ActivationOutcome; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.InOrder; + +class ManagedDeploymentMigrationCommandsTest { + + private static final String OPERATION = "operation-a"; + private static final Instant CREATED = Instant.parse("2026-08-10T01:00:00Z"); + private static final Instant STARTED = CREATED.plusSeconds(1); + + @TempDir + private Path root; + + @Test + void migrateDelegatesWithoutRetainingRequestAndExternalModeIsRejected() { + Fixture fixture = fixture(); + MetadataMigrationRequest request = request(ApplyMode.MANAGED_WRITE); + when(fixture.runner.start(request)).thenReturn(MigrationOperationProjection.view(running())); + + assertThat(fixture.commands.migrate(request).state()).isEqualTo(MigrationOperationState.RUNNING); + assertThatThrownBy(() -> fixture.commands.migrate(request(ApplyMode.EXTERNAL_APPLY))) + .isInstanceOf(MigrationOperationStoreException.class) + .hasNoCause(); + verify(fixture.runner, never()).start(request(ApplyMode.EXTERNAL_APPLY)); + } + + @Test + void statusReadsStoreBeforeRetainedPhaseAndFailsClosedWhileActivationIsPending() { + Fixture fixture = fixture(); + MigrationView view = MigrationOperationProjection.view(activating()); + when(fixture.runner.find(OPERATION)).thenReturn(Optional.of(view)); + when(fixture.coordinator.status()).thenReturn(new RetainedCutoverStatus( + OPERATION, RetainedCutoverStatus.Phase.ACTIVATION_PENDING)); + + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> fixture.commands.migration(OPERATION)); + InOrder order = inOrder(fixture.runner, fixture.coordinator); + order.verify(fixture.runner).find(OPERATION); + order.verify(fixture.coordinator).status(); + } + + @Test + void retainedStatusRequiresAnExactReadyJournalShape() { + Fixture fixture = fixture(); + when(fixture.runner.find(OPERATION)).thenReturn(Optional.empty()); + when(fixture.coordinator.status()).thenReturn(new RetainedCutoverStatus( + OPERATION, RetainedCutoverStatus.Phase.RETAINED)); + + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> fixture.commands.migration(OPERATION)); + + when(fixture.runner.find(OPERATION)).thenReturn( + Optional.of(MigrationOperationProjection.view(awaitingRestart()))); + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> fixture.commands.migration(OPERATION)); + } + + @Test + void awaitingRestartPhaseRejectsReadyJournalShape() { + Fixture fixture = fixture(); + when(fixture.runner.find(OPERATION)).thenReturn( + Optional.of(MigrationOperationProjection.view(ready()))); + when(fixture.coordinator.status()).thenReturn(new RetainedCutoverStatus( + OPERATION, RetainedCutoverStatus.Phase.AWAITING_RESTART_RETAINED)); + + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> fixture.commands.migration(OPERATION)); + } + + @Test + void activationTreatsOwnedMissingJournalAsRecoveryRequired() { + Fixture fixture = fixture(); + when(fixture.coordinator.status()).thenReturn(new RetainedCutoverStatus( + OPERATION, RetainedCutoverStatus.Phase.RETAINED)); + + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> fixture.commands.activate( + OPERATION, + new ActivateMigrationRequest(MigrationOperationState.READY_TO_ACTIVATE))); + } + + @Test + void activationKeepsOperationNotFoundForAnUnownedMissingJournal() { + Fixture fixture = fixture(); + when(fixture.coordinator.status()).thenReturn(RetainedCutoverStatus.empty()); + + assertStoreError(SetupErrorCode.OPERATION_NOT_FOUND, + () -> fixture.commands.activate( + OPERATION, + new ActivateMigrationRequest(MigrationOperationState.READY_TO_ACTIVATE))); + } + + @Test + void readyActivationBindsTheExactDurableCallbackAndReturnsConfirmedAwaitingRestart() throws Exception { + Fixture fixture = fixture(); + advanceToReady(fixture.store); + when(fixture.configuration.activateExact(any(), any())).thenReturn(ActivationOutcome.ACTIVATED); + when(fixture.runner.find(OPERATION)).thenAnswer(ignored -> fixture.store.find(OPERATION) + .map(MigrationOperationProjection::view)); + when(fixture.coordinator.status()) + .thenReturn(new RetainedCutoverStatus(OPERATION, RetainedCutoverStatus.Phase.RETAINED)) + .thenReturn(new RetainedCutoverStatus( + OPERATION, RetainedCutoverStatus.Phase.AWAITING_RESTART_RETAINED)); + when(fixture.coordinator.activateRetained(any(), any())).thenAnswer(invocation -> { + RetainedManagedActivation activation = invocation.getArgument(1); + activation.activate(new RetainedManagedActivationContext(OPERATION, "a".repeat(64))); + return new RetainedManagedActivationResult( + OPERATION, "a".repeat(64), RetainedManagedActivationResult.Status.ACTIVATED); + }); + + MigrationView result = fixture.commands.activate( + OPERATION, new ActivateMigrationRequest(MigrationOperationState.READY_TO_ACTIVATE)); + + assertThat(result.state()).isEqualTo(MigrationOperationState.AWAITING_RESTART); + assertThat(result.restartRequired()).isTrue(); + verify(fixture.configuration).activateExact(any(), any()); + } + + @Test + void activationPendingRetriesOnlyTheAlreadyBoundCallback() { + Fixture fixture = fixture(); + advanceToReady(fixture.store); + fixture.store.compareAndTransition( + OPERATION, MigrationOperationState.READY_TO_ACTIVATE, activating()); + when(fixture.coordinator.status()) + .thenReturn(new RetainedCutoverStatus( + OPERATION, RetainedCutoverStatus.Phase.ACTIVATION_PENDING)) + .thenReturn(new RetainedCutoverStatus( + OPERATION, RetainedCutoverStatus.Phase.AWAITING_RESTART_RETAINED)); + doAnswer(ignored -> { + fixture.store.compareAndTransition( + OPERATION, MigrationOperationState.RUNNING, awaitingRestart()); + return new RetainedManagedActivationResult( + OPERATION, "a".repeat(64), RetainedManagedActivationResult.Status.ACTIVATED); + }).when(fixture.coordinator).retryActivation(OPERATION); + + assertThat(fixture.commands.activate( + OPERATION, new ActivateMigrationRequest(MigrationOperationState.READY_TO_ACTIVATE)).state()) + .isEqualTo(MigrationOperationState.AWAITING_RESTART); + verify(fixture.coordinator, never()).activateRetained(any(), any()); + } + + @Test + void awaitingRestartReplayUsesOnlyTheBoundActivationAndConfirmedJournal() throws Exception { + Fixture fixture = fixture(); + advanceToReady(fixture.store); + fixture.store.compareAndTransition( + OPERATION, MigrationOperationState.READY_TO_ACTIVATE, activating()); + fixture.store.compareAndTransition( + OPERATION, MigrationOperationState.RUNNING, awaitingRestart()); + when(fixture.coordinator.status()) + .thenReturn(new RetainedCutoverStatus( + OPERATION, RetainedCutoverStatus.Phase.AWAITING_RESTART_RETAINED)); + when(fixture.coordinator.retryActivation(OPERATION)).thenReturn( + new RetainedManagedActivationResult( + OPERATION, "a".repeat(64), + RetainedManagedActivationResult.Status.ALREADY_AWAITING_RESTART)); + + MigrationView view = fixture.commands.activate( + OPERATION, new ActivateMigrationRequest(MigrationOperationState.READY_TO_ACTIVATE)); + + assertThat(view.state()).isEqualTo(MigrationOperationState.AWAITING_RESTART); + verify(fixture.coordinator).retryActivation(OPERATION); + verify(fixture.coordinator, never()).activateRetained(any(), any()); + verify(fixture.configuration, never()).activateExact(any(), any()); + } + + @Test + void activeOperationCombinesDurableRunnerViewWithRetainedOwnershipFailClosed() { + Fixture fixture = fixture(); + when(fixture.runner.activeOperationId()).thenReturn(Optional.of(OPERATION)); + when(fixture.coordinator.status()).thenReturn(new RetainedCutoverStatus( + OPERATION, RetainedCutoverStatus.Phase.HANDOFF_PENDING)); + + assertThat(fixture.commands.activeOperationId()).contains(OPERATION); + InOrder order = inOrder(fixture.runner, fixture.coordinator); + order.verify(fixture.runner).activeOperationId(); + order.verify(fixture.coordinator).status(); + + when(fixture.coordinator.status()).thenReturn(new RetainedCutoverStatus( + "operation-b", RetainedCutoverStatus.Phase.RETAINED)); + assertStoreError(SetupErrorCode.OPERATION_CONFLICT, + fixture.commands::activeOperationId); + } + + @Test + void closeOrdersRunnerBeforePhaseAwareShutdownAndNeverReleasesAwaitingRestartFence() { + Fixture fixture = fixture(); + when(fixture.coordinator.status()).thenReturn(new RetainedCutoverStatus( + OPERATION, RetainedCutoverStatus.Phase.AWAITING_RESTART_RETAINED)); + + fixture.commands.close(); + + InOrder order = inOrder(fixture.runner, fixture.coordinator); + order.verify(fixture.runner).close(); + order.verify(fixture.coordinator).shutdownOperation(OPERATION); + verify(fixture.coordinator, never()).releaseRetained(OPERATION); + } + + @Test + void closedFacadeRejectsCommandsWithoutReadingStoreOrRetainedState() { + Fixture fixture = fixture(); + when(fixture.coordinator.status()).thenReturn(RetainedCutoverStatus.empty()); + fixture.commands.close(); + + assertStoreError(SetupErrorCode.MIGRATION_UNAVAILABLE, + () -> fixture.commands.migration(OPERATION)); + assertStoreError(SetupErrorCode.MIGRATION_UNAVAILABLE, + fixture.commands::activeOperationId); + + verify(fixture.runner, never()).find(any()); + } + + private Fixture fixture() { + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + DeploymentMigrationCommandRunner runner = mock(DeploymentMigrationCommandRunner.class); + RetainedCutoverCoordinator coordinator = mock(RetainedCutoverCoordinator.class); + ManagedMigrationConfigurationTransaction configuration = + mock(ManagedMigrationConfigurationTransaction.class); + return new Fixture(store, runner, coordinator, configuration, + new ManagedDeploymentMigrationCommands(runner, store, configuration, coordinator)); + } + + private static MetadataMigrationRequest request(ApplyMode mode) { + return new MetadataMigrationRequest( + OPERATION, MigrationTarget.POSTGRESQL, + new MetadataDatabaseConfiguration( + MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://db.example/hertzbeat", "migration", "password"), + mode); + } + + private static MigrationOperationSnapshot running() { + return snapshot(MigrationOperationState.RUNNING, MigrationStage.COPYING, 0, + VerificationState.PENDING, false, false); + } + + private static MigrationOperationSnapshot ready() { + return snapshot(MigrationOperationState.READY_TO_ACTIVATE, MigrationStage.READY_TO_ACTIVATE, 100, + VerificationState.SUCCEEDED, true, false); + } + + private static MigrationOperationSnapshot activating() { + return snapshot(MigrationOperationState.RUNNING, MigrationStage.ACTIVATING, 100, + VerificationState.SUCCEEDED, false, false); + } + + private static MigrationOperationSnapshot awaitingRestart() { + return snapshot(MigrationOperationState.AWAITING_RESTART, MigrationStage.AWAITING_RESTART, 100, + VerificationState.SUCCEEDED, false, true); + } + + private static MigrationOperationSnapshot verifying() { + return snapshot(MigrationOperationState.RUNNING, MigrationStage.VERIFYING, 100, + VerificationState.RUNNING, false, false); + } + + private static MigrationOperationSnapshot pending() { + return new MigrationOperationSnapshot( + OPERATION, MigrationOperationState.PENDING, MigrationTarget.POSTGRESQL, + ApplyMode.MANAGED_WRITE, MigrationStage.QUEUED, 0, CREATED, null, null, + VerificationState.PENDING, null, null, 250, false, false, false, + "a".repeat(64), "generation-a"); + } + + private static void advanceToReady(FileMigrationOperationStore store) { + store.create(pending()); + store.compareAndTransition(OPERATION, MigrationOperationState.PENDING, running()); + store.compareAndTransition(OPERATION, MigrationOperationState.RUNNING, verifying()); + store.compareAndTransition(OPERATION, MigrationOperationState.RUNNING, ready()); + } + + private static MigrationOperationSnapshot snapshot( + MigrationOperationState state, MigrationStage stage, int progress, + VerificationState verification, boolean activationAvailable, boolean restartRequired) { + int pollAfterMillis = state == MigrationOperationState.RUNNING + || state == MigrationOperationState.AWAITING_RESTART ? 250 : 0; + return new MigrationOperationSnapshot( + OPERATION, state, MigrationTarget.POSTGRESQL, ApplyMode.MANAGED_WRITE, stage, progress, + CREATED, STARTED, null, verification, null, null, pollAfterMillis, + activationAvailable, restartRequired, false, "a".repeat(64), "generation-a"); + } + + private static void assertStoreError(SetupErrorCode code, Runnable action) { + assertThatThrownBy(action::run) + .isInstanceOfSatisfying(MigrationOperationStoreException.class, + failure -> assertThat(failure.errorCode()).isEqualTo(code)) + .hasNoCause(); + } + + private record Fixture( + FileMigrationOperationStore store, + DeploymentMigrationCommandRunner runner, + RetainedCutoverCoordinator coordinator, + ManagedMigrationConfigurationTransaction configuration, + ManagedDeploymentMigrationCommands commands) { } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverShutdownTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverShutdownTest.java new file mode 100644 index 0000000000..c551f15204 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverShutdownTest.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceLease; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceException; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceOrchestrator; +import org.apache.hertzbeat.manager.maintenance.MigrationSourceAction; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.junit.jupiter.api.Test; + +class RetainedCutoverShutdownTest { + + private static final String OPERATION = "operation-a"; + private static final String IDENTITY = "a".repeat(64); + private static final MetadataDatabaseSettings TARGET = new MetadataDatabaseSettings( + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db.example/hertzbeat", "migration"); + private static final Duration TIMEOUT = Duration.ofSeconds(1); + + @Test + void retainedShutdownReleasesExactMaintenanceOnce() { + Fixture fixture = new Fixture(); + fixture.retain(context -> RetainedCopyJournalDisposition.TRANSITIONED); + + fixture.coordinator.shutdownOperation(OPERATION); + fixture.coordinator.shutdownOperation(OPERATION); + + verify(fixture.maintenanceLease, times(1)).close(); + assertThat(fixture.coordinator.status()).isEqualTo(RetainedCutoverStatus.empty()); + } + + @Test + void pendingHandoffIsRetriedBeforeMaintenanceReleaseWithoutRecopy() { + Fixture fixture = new Fixture(); + RetainedCopyJournalHandoff handoff = mock(RetainedCopyJournalHandoff.class); + when(handoff.handoff(any())) + .thenThrow(new RetainedCopyJournalHandoffException(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)) + .thenReturn(RetainedCopyJournalDisposition.ALREADY_CONFIRMED); + assertThatThrownBy(() -> fixture.retain(handoff)) + .isInstanceOf(RetainedCopyJournalHandoffException.class); + + fixture.coordinator.shutdownOperation(OPERATION); + + verify(handoff, times(2)).handoff(any()); + verify(fixture.executor, times(1)).execute(any(), any(), any(), anyDeadline(), any()); + verify(fixture.maintenanceLease, times(1)).close(); + } + + @Test + void pendingActivationRetriesBoundCallbackAndNeverReleasesAwaitingRestartFence() { + Fixture fixture = new Fixture(); + fixture.retain(context -> RetainedCopyJournalDisposition.TRANSITIONED); + RetainedManagedActivation activation = mock(RetainedManagedActivation.class); + when(activation.activate(any())) + .thenThrow(new RetainedManagedActivationException(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)) + .thenReturn(RetainedManagedActivationDisposition.ACTIVATED); + assertThatThrownBy(() -> fixture.coordinator.activateRetained(OPERATION, activation)) + .isInstanceOf(RetainedManagedActivationException.class); + + fixture.coordinator.shutdownOperation(OPERATION); + + assertThat(fixture.coordinator.status().phase()) + .isEqualTo(RetainedCutoverStatus.Phase.AWAITING_RESTART_RETAINED); + verify(activation, times(2)).activate(any()); + verify(fixture.maintenanceLease, never()).close(); + } + + @Test + void awaitingRestartShutdownNeverReleasesFenceOrCallsActivationAgain() { + Fixture fixture = new Fixture(); + fixture.retain(context -> RetainedCopyJournalDisposition.TRANSITIONED); + RetainedManagedActivation activation = mock(RetainedManagedActivation.class); + when(activation.activate(any())).thenReturn(RetainedManagedActivationDisposition.ACTIVATED); + fixture.coordinator.activateRetained(OPERATION, activation); + + fixture.coordinator.shutdownOperation(OPERATION); + + verify(activation, times(1)).activate(any()); + verify(fixture.maintenanceLease, never()).close(); + } + + @Test + void awaitingRestartFenceStillRejectsForeignShutdown() { + Fixture fixture = new Fixture(); + fixture.retain(context -> RetainedCopyJournalDisposition.TRANSITIONED); + RetainedManagedActivation activation = mock(RetainedManagedActivation.class); + when(activation.activate(any())).thenReturn(RetainedManagedActivationDisposition.ACTIVATED); + fixture.coordinator.activateRetained(OPERATION, activation); + + assertThatThrownBy(() -> fixture.coordinator.shutdownOperation("operation-b")) + .isInstanceOf(MigrationMaintenanceException.class) + .hasNoCause(); + + verify(fixture.maintenanceLease, never()).close(); + } + + @Test + void failedReleaseRemainsExactlyRetryable() { + Fixture fixture = new Fixture(); + fixture.retain(context -> RetainedCopyJournalDisposition.TRANSITIONED); + doThrow(new IllegalStateException("private-release")) + .doNothing().when(fixture.maintenanceLease).close(); + + assertThatThrownBy(() -> fixture.coordinator.shutdownOperation(OPERATION)) + .isInstanceOf(RetainedCutoverReleaseRequiredException.class) + .hasNoCause(); + fixture.coordinator.shutdownOperation(OPERATION); + + verify(fixture.maintenanceLease, times(2)).close(); + assertThat(fixture.coordinator.status()).isEqualTo(RetainedCutoverStatus.empty()); + } + + private static JdbcMetadataMigrationDeadline anyDeadline() { + return any(JdbcMetadataMigrationDeadline.class); + } + + private static final class Fixture { + + private final Connection targetConnection = mock(Connection.class); + private final Connection sourceConnection = mock(Connection.class); + private final TargetJdbcConnectionFactory factory = mock(TargetJdbcConnectionFactory.class); + private final TargetJdbcConnectionLease provisionLease = mock(TargetJdbcConnectionLease.class); + private final TargetJdbcConnectionLease copyLease = mock(TargetJdbcConnectionLease.class); + private final FlywayTargetSchemaProvisioner provisioner = mock(FlywayTargetSchemaProvisioner.class); + private final MigrationMaintenanceOrchestrator maintenance = mock(MigrationMaintenanceOrchestrator.class); + private final MigrationMaintenanceLease maintenanceLease = mock(MigrationMaintenanceLease.class); + private final JdbcMetadataMigrationExecutor executor = mock(JdbcMetadataMigrationExecutor.class); + private final SecretValue password = mock(SecretValue.class); + private final RetainedCutoverCoordinator coordinator; + + private Fixture() { + when(provisionLease.targetIdentityHash()).thenReturn(IDENTITY); + when(copyLease.targetIdentityHash()).thenReturn(IDENTITY); + when(factory.acquire(same(TARGET), same(password), anyDeadline())) + .thenReturn(provisionLease, copyLease); + scopedTarget(provisionLease); + scopedTarget(copyLease); + scopedSource(maintenanceLease); + when(provisioner.provision(any(), any(), anyDeadline())).thenReturn( + new TargetSchemaProvisioningOutcome(TargetSchemaConnectionDisposition.REUSABLE)); + when(maintenance.acquire(eq(OPERATION), any())).thenReturn(maintenanceLease); + coordinator = new RetainedCutoverCoordinator( + factory, provisioner, maintenance, executor, new AtomicLong()::get); + } + + private void retain(RetainedCopyJournalHandoff handoff) { + coordinator.execute(OPERATION, TARGET, password, TIMEOUT, + MetadataMigrationProgressSink.NO_OP, RetainedCutoverPreparation.NO_OP, handoff); + } + + private void scopedTarget(TargetJdbcConnectionLease lease) { + doAnswer(invocation -> { + TargetJdbcConnectionAction action = invocation.getArgument(0); + action.execute(targetConnection); + return null; + }).when(lease).withConnection(any()); + } + + private void scopedSource(MigrationMaintenanceLease lease) { + doAnswer(invocation -> { + MigrationSourceAction action = invocation.getArgument(0); + action.execute(sourceConnection); + return null; + }).when(lease).withSourceConnection(any()); + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverStatusTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverStatusTest.java new file mode 100644 index 0000000000..c95ef085c5 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/RetainedCutoverStatusTest.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; + +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceException; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceLease; +import org.junit.jupiter.api.Test; + +class RetainedCutoverStatusTest { + + private static final String OPERATION = "operation-a"; + private static final String IDENTITY = "a".repeat(64); + + @Test + void queryIsTotalAndSecretFreeAcrossEveryOwnedPhase() { + assertThat(new RetainedCutoverState().status()).isEqualTo(RetainedCutoverStatus.empty()); + assertThat(RetainedCutoverStatus.class.getRecordComponents()) + .extracting(component -> component.getName()) + .containsExactly("operationId", "phase"); + + assertThat(executing().state().status().phase()).isEqualTo(RetainedCutoverStatus.Phase.EXECUTING); + assertThat(handoffing().state().status().phase()) + .isEqualTo(RetainedCutoverStatus.Phase.HANDOFFING); + assertThat(handoffPending().state().status().phase()) + .isEqualTo(RetainedCutoverStatus.Phase.HANDOFF_PENDING); + assertThat(retained().state().status().phase()).isEqualTo(RetainedCutoverStatus.Phase.RETAINED); + assertThat(activating().state().status().phase()) + .isEqualTo(RetainedCutoverStatus.Phase.ACTIVATING); + assertThat(activationPending().state().status().phase()) + .isEqualTo(RetainedCutoverStatus.Phase.ACTIVATION_PENDING); + assertThat(awaitingRestart().state().status().phase()) + .isEqualTo(RetainedCutoverStatus.Phase.AWAITING_RESTART_RETAINED); + assertThat(releasing().state().status().phase()) + .isEqualTo(RetainedCutoverStatus.Phase.RELEASING); + assertThat(releasePending().state().status().phase()) + .isEqualTo(RetainedCutoverStatus.Phase.RELEASE_PENDING); + } + + @Test + void foreignOperationCannotUseStatusAsReleaseCapability() { + RetainedCutoverState state = retained().state(); + + assertThat(state.status().operationId()).isEqualTo(OPERATION); + assertThatThrownBy(() -> state.claimRetainedRelease("operation-b")) + .isInstanceOf(MigrationMaintenanceException.class) + .hasNoCause(); + } + + private static TestState executing() { + RetainedCutoverState state = new RetainedCutoverState(); + RetainedCutoverState.Execution execution = + state.reserve(OPERATION, context -> RetainedCopyJournalDisposition.TRANSITIONED); + execution.targetIdentityHash(IDENTITY); + return new TestState(state, execution); + } + + private static TestState handoffPending() { + TestState test = handoffing(); + test.state().handoffPending(test.execution()); + return test; + } + + private static TestState handoffing() { + TestState test = executing(); + test.state().beginHandoff(test.execution(), mock(MigrationMaintenanceLease.class)); + return test; + } + + private static TestState retained() { + TestState test = executing(); + test.state().beginHandoff(test.execution(), mock(MigrationMaintenanceLease.class)); + test.state().completeHandoff(test.execution(), RetainedCopyJournalDisposition.TRANSITIONED); + return test; + } + + private static TestState activationPending() { + TestState test = activating(); + test.state().activationPending(test.execution()); + return test; + } + + private static TestState activating() { + TestState test = retained(); + RetainedManagedActivationClaim claim = test.state().claimManagedActivation( + OPERATION, context -> RetainedManagedActivationDisposition.ACTIVATED); + return new TestState(test.state(), claim.execution()); + } + + private static TestState awaitingRestart() { + TestState test = retained(); + RetainedManagedActivationClaim claim = test.state().claimManagedActivation( + OPERATION, context -> RetainedManagedActivationDisposition.ACTIVATED); + test.state().completeActivation(claim.execution(), RetainedManagedActivationDisposition.ACTIVATED); + return test; + } + + private static TestState releasePending() { + TestState test = releasing(); + test.state().releasePending(test.execution(), test.execution().release()); + return test; + } + + private static TestState releasing() { + TestState test = retained(); + return new TestState(test.state(), test.state().claimRetainedRelease(OPERATION)); + } + + private record TestState( + RetainedCutoverState state, RetainedCutoverState.Execution execution) { } +} From 9217f669adcb4989f61d215e46102fe4d24e3765 Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 22:12:30 +0800 Subject: [PATCH 68/71] Inspect migration targets safely --- .../setup/workflow/FlywaySchemaHistory.java | 113 +++++++- .../MetadataMigrationTargetInspector.java | 179 +++++++++++++ .../TargetSchemaReadOnlyInspector.java | 128 +++++++++ .../MetadataMigrationTargetInspectorTest.java | 203 ++++++++++++++ .../TargetSchemaReadOnlyInspectorTest.java | 248 ++++++++++++++++++ 5 files changed, 863 insertions(+), 8 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationTargetInspector.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaReadOnlyInspector.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationTargetInspectorTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaReadOnlyInspectorTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywaySchemaHistory.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywaySchemaHistory.java index b7f9fdd90d..e4f59741d3 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywaySchemaHistory.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FlywaySchemaHistory.java @@ -32,6 +32,13 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseK final class FlywaySchemaHistory { private static final String TABLE = "flyway_schema_history"; + private static final Set MYSQL_HISTORY_INDEXES = + Set.of("primary", "flyway_schema_history_s_idx"); + private static final Set MYSQL_CONTRACT_INDEXES = Set.of("primary"); + private static final Set POSTGRESQL_HISTORY_INDEXES = + Set.of("flyway_schema_history_pk", "flyway_schema_history_s_idx"); + private static final Set POSTGRESQL_CONTRACT_INDEXES = + Set.of("flyway_schema_contract_pk"); private final MetadataDatabaseKind kind; FlywaySchemaHistory(MetadataDatabaseKind kind) { @@ -88,7 +95,7 @@ final class FlywaySchemaHistory { } void requireEmptyTarget(Connection connection, TargetSchemaJdbcBudget budget) throws SQLException { - if (!currentCatalogSchemaObjects(connection, null, budget).isEmpty()) { + if (!catalogSchemaObjects(connection, budget).isEmpty()) { throw unexpectedTargetState(); } } @@ -154,10 +161,81 @@ final class FlywaySchemaHistory { private Set currentBaselineTables( Connection connection, TargetSchemaJdbcBudget budget) throws SQLException { - return currentCatalogSchemaObjects(connection, new String[]{"TABLE"}, budget); + return catalogSchemaTables(connection, budget); } - private Set currentCatalogSchemaObjects( + Set catalogSchemaTables( + Connection connection, TargetSchemaJdbcBudget budget) throws SQLException { + return readCatalogSchemaObjects(connection, new String[]{"TABLE"}, budget).stream() + .map(CatalogObject::name) + .collect(java.util.stream.Collectors.toUnmodifiableSet()); + } + + Set catalogSchemaObjects( + Connection connection, TargetSchemaJdbcBudget budget) throws SQLException { + return readCatalogSchemaObjects(connection, null, budget); + } + + boolean hasExactHousekeepingIndexes( + Connection connection, TargetSchemaJdbcBudget budget) throws SQLException { + budget.check(); + DatabaseMetaData metadata = connection.getMetaData(); + budget.check(); + String catalog = connection.getCatalog(); + budget.check(); + String schema = null; + if (kind == MetadataDatabaseKind.POSTGRESQL) { + schema = connection.getSchema(); + budget.check(); + } + Set historyIndexes = readIndexes(metadata, catalog, schema, TABLE, budget); + Set contractIndexes = + readIndexes(metadata, catalog, schema, TargetSchemaContract.TABLE, budget); + return hasExactHousekeepingIndexes(historyIndexes, contractIndexes); + } + + boolean hasExactHousekeepingIndexes( + Set historyIndexes, Set contractIndexes) { + Set expectedHistory = kind == MetadataDatabaseKind.MYSQL + ? MYSQL_HISTORY_INDEXES : POSTGRESQL_HISTORY_INDEXES; + Set expectedContract = kind == MetadataDatabaseKind.MYSQL + ? MYSQL_CONTRACT_INDEXES : POSTGRESQL_CONTRACT_INDEXES; + return expectedHistory.equals(normalizeIndexes(historyIndexes)) + && expectedContract.equals(normalizeIndexes(contractIndexes)); + } + + private static Set readIndexes( + DatabaseMetaData metadata, + String catalog, + String schema, + String table, + TargetSchemaJdbcBudget budget) throws SQLException { + Set indexes = new HashSet<>(); + budget.check(); + try (ResultSet rows = metadata.getIndexInfo(catalog, schema, table, false, false)) { + budget.check(); + while (rows.next()) { + budget.check(); + short type = rows.getShort("TYPE"); + budget.check(); + String name = rows.getString("INDEX_NAME"); + budget.check(); + if (type != DatabaseMetaData.tableIndexStatistic && name != null) { + indexes.add(name.toLowerCase(Locale.ROOT)); + } + } + } + budget.check(); + return Set.copyOf(indexes); + } + + private static Set normalizeIndexes(Set indexes) { + return indexes.stream() + .map(index -> index.toLowerCase(Locale.ROOT)) + .collect(java.util.stream.Collectors.toUnmodifiableSet()); + } + + private Set readCatalogSchemaObjects( Connection connection, String[] types, TargetSchemaJdbcBudget budget) throws SQLException { @@ -171,16 +249,24 @@ final class FlywaySchemaHistory { schema = connection.getSchema(); budget.check(); } - Set names = new HashSet<>(); - try (ResultSet objects = metadata.getTables(catalog, schema, "%", types)) { + Set catalogObjects = new HashSet<>(); + try (ResultSet rows = metadata.getTables(catalog, schema, "%", types)) { budget.check(); - while (objects.next()) { + while (rows.next()) { budget.check(); - names.add(objects.getString("TABLE_NAME").toLowerCase(Locale.ROOT)); + String name = rows.getString("TABLE_NAME"); + budget.check(); + String type = types == null ? rows.getString("TABLE_TYPE") : types[0]; + budget.check(); + if (name == null || type == null) { + throw unexpectedTargetState(); + } + catalogObjects.add(new CatalogObject( + name.toLowerCase(Locale.ROOT), type.toUpperCase(Locale.ROOT))); } } budget.check(); - return Set.copyOf(names); + return Set.copyOf(catalogObjects); } private static String abbreviate(String value, int maximumLength) { @@ -190,4 +276,15 @@ final class FlywaySchemaHistory { private static SQLException unexpectedTargetState() { return new SQLException("Target schema is not empty or does not contain the current baseline", "55000"); } + + record CatalogObject(String name, String type) { + + CatalogObject { + if (name == null || name.isBlank() || type == null || type.isBlank()) { + throw new IllegalArgumentException("Catalog object fields must not be blank"); + } + name = name.toLowerCase(Locale.ROOT); + type = type.toUpperCase(Locale.ROOT); + } + } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationTargetInspector.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationTargetInspector.java new file mode 100644 index 0000000000..e8f13e1fbb --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationTargetInspector.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.Objects; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.TargetInspection; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; + +/** Borrows the one target factory and retains only exact unresolved cleanup ownership. */ +final class MetadataMigrationTargetInspector { + + private final TargetJdbcConnectionFactory factory; + private final TargetSchemaReadOnlyInspector schema; + private TargetJdbcConnectionLease pendingLease; + private Error pendingFatal; + private boolean pendingAcquire; + private boolean shutdown; + + MetadataMigrationTargetInspector( + TargetJdbcConnectionFactory factory, + TargetSchemaReadOnlyInspector schema) { + this.factory = Objects.requireNonNull(factory, "factory"); + this.schema = Objects.requireNonNull(schema, "schema"); + } + + synchronized TargetInspection inspect( + MetadataDatabaseSettings settings, + SecretValue borrowedPassword, + JdbcMetadataMigrationDeadline deadline) { + Objects.requireNonNull(settings, "settings"); + Objects.requireNonNull(borrowedPassword, "borrowedPassword"); + Objects.requireNonNull(deadline, "deadline"); + if (shutdown || hasPendingCleanup()) { + return TargetInspection.UNKNOWN; + } + TargetJdbcConnectionLease lease; + try { + lease = factory.acquire(settings, borrowedPassword, deadline); + } catch (TargetJdbcConnectionException failure) { + pendingAcquire = mayOwnFailedAcquire(failure.code()); + return TargetInspection.UNKNOWN; + } catch (MetadataMigrationException failure) { + pendingAcquire = true; + return TargetInspection.UNKNOWN; + } catch (RuntimeException failure) { + pendingAcquire = true; + return TargetInspection.UNKNOWN; + } catch (Error fatal) { + pendingAcquire = true; + pendingFatal = fatal; + throw fatal; + } + InspectionHolder result = inspectLease(lease, settings, deadline); + return closeLease(lease, result); + } + + synchronized void retryCleanup(JdbcMetadataMigrationDeadline deadline) { + Objects.requireNonNull(deadline, "deadline"); + if (pendingLease != null) { + retryLeaseClose(); + return; + } + if (!pendingAcquire) { + return; + } + try { + factory.settleFailedAcquire(deadline); + } catch (RuntimeException failure) { + throw new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED); + } catch (Error fatal) { + if (pendingFatal == null) { + pendingFatal = fatal; + } + throw pendingFatal; + } + Error fatal = pendingFatal; + pendingAcquire = false; + pendingFatal = null; + if (fatal != null) { + throw fatal; + } + } + + synchronized void shutdown(JdbcMetadataMigrationDeadline deadline) { + shutdown = true; + retryCleanup(deadline); + } + + @Override + public synchronized String toString() { + return "MetadataMigrationTargetInspector[shutdown=" + shutdown + + ", pendingCleanup=" + hasPendingCleanup() + ']'; + } + + private InspectionHolder inspectLease( + TargetJdbcConnectionLease lease, + MetadataDatabaseSettings settings, + JdbcMetadataMigrationDeadline deadline) { + InspectionHolder result = new InspectionHolder(); + try { + lease.withConnection(connection -> result.inspection = + schema.inspect(connection, settings.kind(), deadline)); + } catch (RuntimeException failure) { + result.inspection = TargetInspection.UNKNOWN; + } catch (Error fatal) { + result.fatal = fatal; + } + return result; + } + + private TargetInspection closeLease( + TargetJdbcConnectionLease lease, + InspectionHolder result) { + try { + lease.close(); + } catch (RuntimeException failure) { + retainLease(lease, result.fatal); + if (result.fatal != null) { + throw result.fatal; + } + return TargetInspection.UNKNOWN; + } catch (Error fatal) { + Error primary = result.fatal == null ? fatal : result.fatal; + retainLease(lease, primary); + throw primary; + } + if (result.fatal != null) { + throw result.fatal; + } + return result.inspection; + } + + private void retryLeaseClose() { + try { + pendingLease.close(); + } catch (RuntimeException failure) { + throw new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED); + } catch (Error fatal) { + if (pendingFatal == null) { + pendingFatal = fatal; + } + throw pendingFatal; + } + Error fatal = pendingFatal; + pendingLease = null; + pendingFatal = null; + if (fatal != null) { + throw fatal; + } + } + + private void retainLease(TargetJdbcConnectionLease lease, Error fatal) { + pendingLease = lease; + pendingFatal = fatal; + } + + private boolean hasPendingCleanup() { + return pendingLease != null || pendingAcquire; + } + + private static boolean mayOwnFailedAcquire(TargetJdbcConnectionErrorCode code) { + return switch (code) { + case TIMEOUT, UNAVAILABLE, CLEANUP_REQUIRED -> true; + case TARGET_MISMATCH, OPERATION_CONFLICT, FACTORY_CLOSED -> false; + }; + } + + private static final class InspectionHolder { + + private TargetInspection inspection = TargetInspection.UNKNOWN; + private Error fatal; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaReadOnlyInspector.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaReadOnlyInspector.java new file mode 100644 index 0000000000..9881a71fb3 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaReadOnlyInspector.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.TargetInspection; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.workflow.FlywaySchemaHistory.CatalogObject; + +/** Classifies a target schema without mutating it or retaining its connection. */ +final class TargetSchemaReadOnlyInspector { + + private static final String HISTORY_TABLE = "flyway_schema_history"; + private static final Set CURRENT_OBJECT_TYPES = Set.of("TABLE", "INDEX", "SEQUENCE"); + + private final BaselineLoader baselines; + private final HistoryFactory histories; + + TargetSchemaReadOnlyInspector() { + this(TargetSchemaBaseline::load, FlywaySchemaHistory::new); + } + + TargetSchemaReadOnlyInspector(BaselineLoader baselines, HistoryFactory histories) { + this.baselines = Objects.requireNonNull(baselines, "baselines"); + this.histories = Objects.requireNonNull(histories, "histories"); + } + + TargetInspection inspect( + Connection connection, + MetadataDatabaseKind kind, + JdbcMetadataMigrationDeadline deadline) { + Objects.requireNonNull(connection, "connection"); + Objects.requireNonNull(kind, "kind"); + Objects.requireNonNull(deadline, "deadline"); + if (kind == MetadataDatabaseKind.H2) { + return TargetInspection.UNKNOWN; + } + TargetSchemaJdbcBudget budget = new TargetSchemaJdbcBudget(deadline); + try { + TargetSchemaBaseline baseline = baselines.load(kind); + FlywaySchemaHistory history = histories.create(kind); + Set catalogObjects = history.catalogSchemaObjects(connection, budget); + if (catalogObjects.isEmpty()) { + return TargetInspection.EMPTY; + } + if (containsUnsupportedRelation(catalogObjects)) { + return TargetInspection.NON_EMPTY; + } + Set actualTables = history.catalogSchemaTables(connection, budget); + if (!actualTables.equals(expectedTables(baseline))) { + return TargetInspection.NON_EMPTY; + } + if (!history.isCurrent(connection, baseline, budget)) { + return TargetInspection.NON_EMPTY; + } + if (!history.hasExactHousekeepingIndexes(connection, budget)) { + return TargetInspection.NON_EMPTY; + } + return applicationTablesEmpty(connection, baseline.expectedTables(), budget) + ? TargetInspection.EMPTY + : TargetInspection.NON_EMPTY; + } catch (SQLException failure) { + return "55000".equals(failure.getSQLState()) + ? TargetInspection.NON_EMPTY + : TargetInspection.UNKNOWN; + } catch (IOException failure) { + return TargetInspection.UNKNOWN; + } catch (RuntimeException failure) { + return TargetInspection.UNKNOWN; + } + } + + private static Set expectedTables(TargetSchemaBaseline baseline) { + Set expected = new HashSet<>(baseline.expectedTables()); + expected.add(HISTORY_TABLE); + expected.add(TargetSchemaContract.TABLE); + return Set.copyOf(expected); + } + + private static boolean containsUnsupportedRelation(Set catalogObjects) { + return catalogObjects.stream().map(CatalogObject::type) + .anyMatch(type -> !CURRENT_OBJECT_TYPES.contains(type)); + } + + private static boolean applicationTablesEmpty( + Connection connection, + Set applicationTables, + TargetSchemaJdbcBudget budget) throws SQLException { + try (Statement statement = connection.createStatement()) { + for (String table : applicationTables) { + budget.apply(statement); + try (ResultSet rows = statement.executeQuery("SELECT 1 FROM " + table + " LIMIT 1")) { + budget.check(); + boolean hasRows = rows.next(); + budget.check(); + if (hasRows) { + return false; + } + } + } + } + return true; + } + + @FunctionalInterface + interface BaselineLoader { + + TargetSchemaBaseline load(MetadataDatabaseKind kind) throws IOException; + } + + @FunctionalInterface + interface HistoryFactory { + + FlywaySchemaHistory create(MetadataDatabaseKind kind); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationTargetInspectorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationTargetInspectorTest.java new file mode 100644 index 0000000000..33726f8047 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MetadataMigrationTargetInspectorTest.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Field; +import java.sql.Connection; +import java.time.Duration; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.TargetInspection; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class MetadataMigrationTargetInspectorTest { + + private static final MetadataDatabaseSettings SETTINGS = new MetadataDatabaseSettings( + MetadataDatabaseKind.MYSQL, "jdbc:mysql://db.example/hertzbeat", "operator"); + + private final TargetJdbcConnectionFactory factory = mock(TargetJdbcConnectionFactory.class); + private final TargetJdbcConnectionLease lease = mock(TargetJdbcConnectionLease.class); + private final TargetSchemaReadOnlyInspector schema = mock(TargetSchemaReadOnlyInspector.class); + private final Connection connection = mock(Connection.class); + private SecretValue password; + private MetadataMigrationTargetInspector inspector; + + @BeforeEach + void setUp() { + password = SecretValue.of("borrowed-password"); + inspector = new MetadataMigrationTargetInspector(factory, schema); + when(factory.acquire(eq(SETTINGS), eq(password), any())).thenReturn(lease); + doAnswer(invocation -> { + invocation.getArgument(0).execute(connection); + return null; + }).when(lease).withConnection(any()); + when(schema.inspect(eq(connection), eq(MetadataDatabaseKind.MYSQL), any())) + .thenReturn(TargetInspection.EMPTY); + } + + @AfterEach + void cleanUp() { + Thread.interrupted(); + password.close(); + } + + @Test + void borrowsOneFactoryAndOneDeadlineWithoutConsumingTheCallerSecret() { + JdbcMetadataMigrationDeadline deadline = deadline(); + + assertThat(inspector.inspect(SETTINGS, password, deadline)).isEqualTo(TargetInspection.EMPTY); + + verify(factory).acquire(SETTINGS, password, deadline); + verify(schema).inspect(connection, MetadataDatabaseKind.MYSQL, deadline); + verify(lease).close(); + assertThat(password.copy()).containsExactly("borrowed-password".toCharArray()); + verify(factory, never()).close(); + } + + @Test + void leaseCloseFailureRetainsExactLeaseUntilRetryWithoutReinspection() { + doThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED)) + .doNothing().when(lease).close(); + + assertThat(inspector.inspect(SETTINGS, password, deadline())).isEqualTo(TargetInspection.UNKNOWN); + assertThat(inspector.inspect(SETTINGS, password, deadline())).isEqualTo(TargetInspection.UNKNOWN); + inspector.retryCleanup(deadline()); + + verify(factory, times(1)).acquire(eq(SETTINGS), eq(password), any()); + verify(schema, times(1)).inspect(eq(connection), eq(MetadataDatabaseKind.MYSQL), any()); + verify(lease, times(2)).close(); + } + + @Test + void timedOutAcquireRetainsSettlementButOperationConflictTakesNoOwnership() { + when(factory.acquire(eq(SETTINGS), eq(password), any())) + .thenThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.TIMEOUT)); + when(factory.settleFailedAcquire(any())).thenReturn(TargetJdbcFailedAcquireSettlement.REUSABLE); + + assertThat(inspector.inspect(SETTINGS, password, deadline())).isEqualTo(TargetInspection.UNKNOWN); + inspector.retryCleanup(deadline()); + verify(factory).settleFailedAcquire(any()); + + when(factory.acquire(eq(SETTINGS), eq(password), any())) + .thenThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.OPERATION_CONFLICT)); + assertThat(inspector.inspect(SETTINGS, password, deadline())).isEqualTo(TargetInspection.UNKNOWN); + inspector.retryCleanup(deadline()); + verify(factory, times(1)).settleFailedAcquire(any()); + } + + @Test + void firstInspectionErrorSurvivesUncertainCloseAndExactRetry() { + AssertionError fatal = new AssertionError("private fatal"); + when(schema.inspect(eq(connection), eq(MetadataDatabaseKind.MYSQL), any())).thenThrow(fatal); + doThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED)) + .doNothing().when(lease).close(); + + assertThatThrownBy(() -> inspector.inspect(SETTINGS, password, deadline())).isSameAs(fatal); + assertThatThrownBy(() -> inspector.retryCleanup(deadline())).isSameAs(fatal); + + verify(factory, times(1)).acquire(eq(SETTINGS), eq(password), any()); + verify(schema, times(1)).inspect(eq(connection), eq(MetadataDatabaseKind.MYSQL), any()); + verify(lease, times(2)).close(); + } + + @Test + void interruptedAcquireRemainsUnknownAndPreservesTheInterrupt() { + Thread.currentThread().interrupt(); + when(factory.acquire(eq(SETTINGS), eq(password), any())) + .thenThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.TIMEOUT)); + + assertThat(inspector.inspect(SETTINGS, password, deadline())).isEqualTo(TargetInspection.UNKNOWN); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + assertThat(password.copy()).containsExactly("borrowed-password".toCharArray()); + } + + @Test + void lifecycleStateCannotRetainCredentialsOrTargetSettings() { + assertThat(MetadataMigrationTargetInspector.class.getDeclaredFields()) + .extracting(Field::getType) + .doesNotContain(SecretValue.class) + .doesNotContain(MetadataDatabaseSettings.class); + assertThat(inspector.toString()) + .doesNotContain("borrowed-password", "jdbc:mysql", "operator"); + } + + @Test + void shutdownRetriesExactPendingLeaseWithoutClosingBorrowedFactoryOrAllowingNewInspection() { + doThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED)) + .doThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED)) + .doNothing().when(lease).close(); + + assertThat(inspector.inspect(SETTINGS, password, deadline())).isEqualTo(TargetInspection.UNKNOWN); + assertThatThrownBy(() -> inspector.shutdown(deadline())) + .isInstanceOf(TargetJdbcConnectionException.class); + assertThat(inspector.inspect(SETTINGS, password, deadline())).isEqualTo(TargetInspection.UNKNOWN); + inspector.shutdown(deadline()); + + verify(factory, times(1)).acquire(eq(SETTINGS), eq(password), any()); + verify(factory, never()).close(); + verify(lease, times(3)).close(); + } + + @Test + void acquisitionFatalIsReplayedOnceAfterReusableSettlementThenHealthyInspectionCanRun() { + AssertionError fatal = new AssertionError("private acquisition fatal"); + when(factory.acquire(eq(SETTINGS), eq(password), any())) + .thenThrow(fatal) + .thenThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.TIMEOUT)) + .thenReturn(lease); + when(factory.settleFailedAcquire(any())) + .thenReturn(TargetJdbcFailedAcquireSettlement.REUSABLE) + .thenReturn(TargetJdbcFailedAcquireSettlement.REUSABLE); + + assertThatThrownBy(() -> inspector.inspect(SETTINGS, password, deadline())).isSameAs(fatal); + assertThatThrownBy(() -> inspector.retryCleanup(deadline())).isSameAs(fatal); + assertThat(inspector.inspect(SETTINGS, password, deadline())).isEqualTo(TargetInspection.UNKNOWN); + inspector.retryCleanup(deadline()); + assertThat(inspector.inspect(SETTINGS, password, deadline())).isEqualTo(TargetInspection.EMPTY); + inspector.shutdown(deadline()); + + verify(factory, times(3)).acquire(eq(SETTINGS), eq(password), any()); + verify(factory, times(2)).settleFailedAcquire(any()); + } + + @Test + void shutdownRetriesFailedAcquireSettlementWithoutClosingBorrowedFactory() { + when(factory.acquire(eq(SETTINGS), eq(password), any())) + .thenThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.TIMEOUT)); + when(factory.settleFailedAcquire(any())) + .thenThrow(new TargetJdbcConnectionException(TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED)) + .thenReturn(TargetJdbcFailedAcquireSettlement.REUSABLE); + + assertThat(inspector.inspect(SETTINGS, password, deadline())).isEqualTo(TargetInspection.UNKNOWN); + assertThatThrownBy(() -> inspector.shutdown(deadline())) + .isInstanceOf(TargetJdbcConnectionException.class); + inspector.shutdown(deadline()); + + verify(factory, times(2)).settleFailedAcquire(any()); + verify(factory, never()).close(); + } + + private static JdbcMetadataMigrationDeadline deadline() { + return JdbcMetadataMigrationDeadline.start(Duration.ofSeconds(5), System::nanoTime); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaReadOnlyInspectorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaReadOnlyInspectorTest.java new file mode 100644 index 0000000000..22fd9c4339 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/TargetSchemaReadOnlyInspectorTest.java @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.util.HashSet; +import java.util.Set; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.TargetInspection; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.workflow.FlywaySchemaHistory.CatalogObject; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class TargetSchemaReadOnlyInspectorTest { + + private static final Set APPLICATION_TABLES = Set.of("app_one", "app_two"); + + private final Connection connection = mock(Connection.class); + private final FlywaySchemaHistory history = mock(FlywaySchemaHistory.class); + private final Statement statement = mock(Statement.class); + private final ResultSet rows = mock(ResultSet.class); + private TargetSchemaBaseline baseline; + private TargetSchemaReadOnlyInspector inspector; + + @BeforeEach + void setUp() throws Exception { + baseline = mock(TargetSchemaBaseline.class); + when(baseline.expectedTables()).thenReturn(APPLICATION_TABLES); + inspector = new TargetSchemaReadOnlyInspector( + kind -> baseline, + kind -> history); + when(connection.createStatement()).thenReturn(statement); + when(statement.executeQuery(anyString())).thenReturn(rows); + when(rows.next()).thenReturn(false); + when(history.hasExactHousekeepingIndexes(eq(connection), any())).thenReturn(true); + } + + @AfterEach + void clearInterrupt() { + Thread.interrupted(); + } + + @Test + void physicalEmptyCatalogIsApiEmptyWithoutCurrentSchemaQueries() throws Exception { + when(history.catalogSchemaObjects(eq(connection), any())).thenReturn(Set.of()); + when(history.catalogSchemaTables(eq(connection), any())).thenReturn(Set.of()); + + assertThat(inspector.inspect(connection, MetadataDatabaseKind.MYSQL, deadline())) + .isEqualTo(TargetInspection.EMPTY); + + verify(history, never()).isCurrent(eq(connection), eq(baseline), any()); + verify(connection, never()).createStatement(); + } + + @Test + void exactCurrentSchemaRequiresEveryApplicationTableToBeEmpty() throws Exception { + when(history.catalogSchemaObjects(eq(connection), any())).thenReturn(tableObjects()); + when(history.catalogSchemaTables(eq(connection), any())).thenReturn(expectedTables()); + when(history.isCurrent(eq(connection), eq(baseline), any())).thenReturn(true); + + assertThat(inspector.inspect(connection, MetadataDatabaseKind.MYSQL, deadline())) + .isEqualTo(TargetInspection.EMPTY); + + verify(statement, times(baseline.expectedTables().size())).executeQuery(anyString()); + verify(rows, times(baseline.expectedTables().size())).close(); + } + + @Test + void extraOrMissingObjectIsNonEmptyWithoutReadingApplicationRows() throws Exception { + Set extra = new HashSet<>(expectedTables()); + extra.add("foreign_table"); + when(history.catalogSchemaObjects(eq(connection), any())).thenReturn(tableObjects()); + when(history.catalogSchemaTables(eq(connection), any())).thenReturn(Set.copyOf(extra)); + + assertThat(inspector.inspect(connection, MetadataDatabaseKind.MYSQL, deadline())) + .isEqualTo(TargetInspection.NON_EMPTY); + + verify(history, never()).isCurrent(eq(connection), eq(baseline), any()); + verify(connection, never()).createStatement(); + } + + @Test + void currentSchemaWithAnyApplicationRowIsNonEmpty() throws Exception { + when(history.catalogSchemaObjects(eq(connection), any())).thenReturn(tableObjects()); + when(history.catalogSchemaTables(eq(connection), any())).thenReturn(expectedTables()); + when(history.isCurrent(eq(connection), eq(baseline), any())).thenReturn(true); + when(rows.next()).thenReturn(true); + + assertThat(inspector.inspect(connection, MetadataDatabaseKind.MYSQL, deadline())) + .isEqualTo(TargetInspection.NON_EMPTY); + } + + @Test + void readableCurrentContractMismatchIsNonEmptyRatherThanUnknown() throws Exception { + when(history.catalogSchemaObjects(eq(connection), any())).thenReturn(tableObjects()); + when(history.catalogSchemaTables(eq(connection), any())).thenReturn(expectedTables()); + when(history.isCurrent(eq(connection), eq(baseline), any())) + .thenThrow(new SQLException("private shape", "55000")); + + assertThat(inspector.inspect(connection, MetadataDatabaseKind.MYSQL, deadline())) + .isEqualTo(TargetInspection.NON_EMPTY); + } + + @Test + void unreadableOrInterruptedInspectionIsUnknownAndPreservesInterrupt() throws Exception { + when(history.catalogSchemaObjects(eq(connection), any())) + .thenThrow(new SQLException("private target", "42501")); + assertThat(inspector.inspect(connection, MetadataDatabaseKind.MYSQL, deadline())) + .isEqualTo(TargetInspection.UNKNOWN); + + Thread.currentThread().interrupt(); + assertThat(inspector.inspect(connection, MetadataDatabaseKind.MYSQL, deadline())) + .isEqualTo(TargetInspection.UNKNOWN); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } + + @Test + void viewOnlyAndCurrentSchemaWithForeignViewAreNonEmpty() throws Exception { + CatalogObject view = new CatalogObject("foreign_view", "VIEW"); + when(history.catalogSchemaObjects(eq(connection), any())).thenReturn(Set.of(view)); + when(history.catalogSchemaTables(eq(connection), any())).thenReturn(Set.of()); + assertThat(inspector.inspect(connection, MetadataDatabaseKind.MYSQL, deadline())) + .isEqualTo(TargetInspection.NON_EMPTY); + + Set currentWithMaterialized = new HashSet<>(tableObjects()); + currentWithMaterialized.add(new CatalogObject("foreign_materialized", "MATERIALIZED VIEW")); + when(history.catalogSchemaObjects(eq(connection), any())) + .thenReturn(Set.copyOf(currentWithMaterialized)); + assertThat(inspector.inspect(connection, MetadataDatabaseKind.POSTGRESQL, deadline())) + .isEqualTo(TargetInspection.NON_EMPTY); + + when(history.catalogSchemaObjects(eq(connection), any())) + .thenReturn(Set.of(new CatalogObject("foreign_seq", "SEQUENCE"))); + assertThat(inspector.inspect(connection, MetadataDatabaseKind.POSTGRESQL, deadline())) + .isEqualTo(TargetInspection.NON_EMPTY); + + Set currentWithView = new HashSet<>(tableObjects()); + currentWithView.add(view); + when(history.catalogSchemaObjects(eq(connection), any())).thenReturn(Set.copyOf(currentWithView)); + when(history.catalogSchemaTables(eq(connection), any())).thenReturn(expectedTables()); + when(history.isCurrent(eq(connection), eq(baseline), any())).thenReturn(true); + assertThat(inspector.inspect(connection, MetadataDatabaseKind.MYSQL, deadline())) + .isEqualTo(TargetInspection.NON_EMPTY); + } + + @Test + void postRowDeadlineCheckAppliesToEmptyAndNonEmptyResults() throws Exception { + when(history.catalogSchemaObjects(eq(connection), any())).thenReturn(tableObjects()); + when(history.catalogSchemaTables(eq(connection), any())).thenReturn(expectedTables()); + when(history.isCurrent(eq(connection), eq(baseline), any())).thenReturn(true); + when(rows.next()).thenAnswer(invocation -> { + Thread.currentThread().interrupt(); + return false; + }); + + assertThat(inspector.inspect(connection, MetadataDatabaseKind.MYSQL, deadline())) + .isEqualTo(TargetInspection.UNKNOWN); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + + Thread.interrupted(); + when(rows.next()).thenAnswer(invocation -> { + Thread.currentThread().interrupt(); + return true; + }); + assertThat(inspector.inspect(connection, MetadataDatabaseKind.MYSQL, deadline())) + .isEqualTo(TargetInspection.UNKNOWN); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } + + @Test + void currentPostgresqlOwnedSequencesAndIndexesRemainEmptyButForeignSequenceDoesNot() + throws Exception { + Set ownedObjects = new HashSet<>(tableObjects()); + ownedObjects.add(new CatalogObject("app_one_id_seq", "SEQUENCE")); + ownedObjects.add(new CatalogObject("app_one_pkey", "INDEX")); + when(history.catalogSchemaObjects(eq(connection), any())).thenReturn(Set.copyOf(ownedObjects)); + when(history.catalogSchemaTables(eq(connection), any())).thenReturn(expectedTables()); + when(history.isCurrent(eq(connection), eq(baseline), any())).thenReturn(true); + assertThat(inspector.inspect(connection, MetadataDatabaseKind.POSTGRESQL, deadline())) + .isEqualTo(TargetInspection.EMPTY); + + ownedObjects.add(new CatalogObject("foreign_seq", "SEQUENCE")); + when(history.catalogSchemaObjects(eq(connection), any())).thenReturn(Set.copyOf(ownedObjects)); + when(history.isCurrent(eq(connection), eq(baseline), any())) + .thenThrow(new SQLException("private sequence mismatch", "55000")); + assertThat(inspector.inspect(connection, MetadataDatabaseKind.POSTGRESQL, deadline())) + .isEqualTo(TargetInspection.NON_EMPTY); + } + + @Test + void housekeepingIndexAllowlistIsVendorExactAndForeignIndexesMakeCurrentSchemaNonEmpty() + throws Exception { + FlywaySchemaHistory mysql = new FlywaySchemaHistory(MetadataDatabaseKind.MYSQL); + assertThat(mysql.hasExactHousekeepingIndexes( + Set.of("primary", "flyway_schema_history_s_idx"), Set.of("primary"))).isTrue(); + assertThat(mysql.hasExactHousekeepingIndexes( + Set.of("primary", "flyway_schema_history_s_idx", "foreign_idx"), Set.of("primary"))).isFalse(); + + FlywaySchemaHistory postgresql = new FlywaySchemaHistory(MetadataDatabaseKind.POSTGRESQL); + assertThat(postgresql.hasExactHousekeepingIndexes( + Set.of("flyway_schema_history_pk", "flyway_schema_history_s_idx"), + Set.of("flyway_schema_contract_pk"))).isTrue(); + + when(history.catalogSchemaObjects(eq(connection), any())).thenReturn(tableObjects()); + when(history.catalogSchemaTables(eq(connection), any())).thenReturn(expectedTables()); + when(history.isCurrent(eq(connection), eq(baseline), any())).thenReturn(true); + when(history.hasExactHousekeepingIndexes(eq(connection), any())).thenReturn(false); + assertThat(inspector.inspect(connection, MetadataDatabaseKind.MYSQL, deadline())) + .isEqualTo(TargetInspection.NON_EMPTY); + } + + private Set expectedTables() { + Set expected = new HashSet<>(APPLICATION_TABLES); + expected.add("flyway_schema_history"); + expected.add(TargetSchemaContract.TABLE); + return Set.copyOf(expected); + } + + private Set tableObjects() { + return expectedTables().stream() + .map(name -> new CatalogObject(name, "TABLE")) + .collect(java.util.stream.Collectors.toUnmodifiableSet()); + } + + private static JdbcMetadataMigrationDeadline deadline() { + return JdbcMetadataMigrationDeadline.start(Duration.ofSeconds(5), System::nanoTime); + } +} From 021cbb176f7bae42bee0280680a1d6ab31c68ff0 Mon Sep 17 00:00:00 2001 From: Logic Date: Mon, 10 Aug 2026 23:23:32 +0800 Subject: [PATCH 69/71] Wire managed metadata migration --- .../setup/api/MigrationContractValidator.java | 6 +- .../setup/security/SecureSetupFileLock.java | 37 ++ .../workflow/DefaultDeploymentWorkflow.java | 297 ++++++++++++ .../DeploymentMigrationCommandRunner.java | 25 ++ .../DeploymentMigrationConfiguration.java | 109 +++++ .../workflow/DeploymentMigrationRuntime.java | 301 +++++++++++++ .../workflow/DeploymentViewProjector.java | 156 +++++++ .../DeploymentWorkflowFailureMapper.java | 116 +++++ .../workflow/FileMigrationOperationStore.java | 33 ++ .../ManagedDeploymentMigrationCommands.java | 79 +++- .../workflow/MigrationPreparationBarrier.java | 15 +- .../workflow/TargetJdbcAbortExecutor.java | 31 ++ .../DefaultDeploymentWorkflowTest.java | 421 ++++++++++++++++++ .../DeploymentMigrationConfigurationTest.java | 153 +++++++ .../DeploymentMigrationRuntimeTest.java | 267 +++++++++++ .../workflow/DeploymentViewProjectorTest.java | 132 ++++++ .../DeploymentWorkflowFailureMapperTest.java | 68 +++ ...anagedDeploymentMigrationCommandsTest.java | 116 ++++- .../ManagedMigrationCommandFlowTest.java | 267 +++++++++++ .../workflow/MigrationTestSnapshots.java | 50 +++ 20 files changed, 2674 insertions(+), 5 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultDeploymentWorkflow.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationConfiguration.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationRuntime.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentViewProjector.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentWorkflowFailureMapper.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcAbortExecutor.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultDeploymentWorkflowTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationConfigurationTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationRuntimeTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentViewProjectorTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentWorkflowFailureMapperTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationCommandFlowTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationTestSnapshots.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/MigrationContractValidator.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/MigrationContractValidator.java index 343325528f..9e86ed99a5 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/MigrationContractValidator.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/api/MigrationContractValidator.java @@ -90,8 +90,10 @@ final class MigrationContractValidator { requireBlocker(capability, SetupErrorCode.MIGRATION_MULTI_NODE_UNSUPPORTED, MaintenanceAdmission.NOT_APPLICABLE); } else if (topology == DeploymentTopology.UNKNOWN) { - requireBlocker(capability, SetupErrorCode.MIGRATION_TOPOLOGY_UNAVAILABLE, - MaintenanceAdmission.NOT_APPLICABLE); + if (capability.blockedBy() != SetupErrorCode.MIGRATION_UNAVAILABLE) { + requireBlocker(capability, SetupErrorCode.MIGRATION_TOPOLOGY_UNAVAILABLE, + MaintenanceAdmission.NOT_APPLICABLE); + } } else { validateSingleNodeAdmission(maintenance, capability); } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileLock.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileLock.java index 6aaa58e26a..dced6381b2 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileLock.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/security/SecureSetupFileLock.java @@ -76,6 +76,31 @@ public final class SecureSetupFileLock { }); } + /** Attempts one operation without waiting for either the in-JVM or OS lock. */ + public TryResult tryExecute(IoOperation operation) throws IOException { + Objects.requireNonNull(operation, "operation"); + if (!jvmLock.tryLock()) { + return TryResult.busy(); + } + try { + LockIdentity identity = initializeAndValidate(); + try (FileChannel channel = FileChannel.open( + lockFile, Set.of(StandardOpenOption.READ, StandardOpenOption.WRITE, + LinkOption.NOFOLLOW_LINKS)); + FileLock acquired = channel.tryLock()) { + if (acquired == null) { + return TryResult.busy(); + } + validateLockedIdentity(channel, identity); + T result = operation.run(); + validateLockedIdentity(channel, identity); + return TryResult.acquired(result); + } + } finally { + jvmLock.unlock(); + } + } + /** Validates an existing lock without creating, replacing, or otherwise mutating it. */ public static boolean isValidExistingLock(Path installationRoot, String relativePath) { Objects.requireNonNull(installationRoot, "installationRoot"); @@ -195,4 +220,16 @@ public final class SecureSetupFileLock { public interface IoAction { void run() throws IOException; } + + /** Non-blocking lock attempt result; a null operation value remains distinguishable from busy. */ + public record TryResult(boolean acquired, T value) { + + static TryResult busy() { + return new TryResult<>(false, null); + } + + static TryResult acquired(T value) { + return new TryResult<>(true, value); + } + } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultDeploymentWorkflow.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultDeploymentWorkflow.java new file mode 100644 index 0000000000..60c832dfae --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DefaultDeploymentWorkflow.java @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.time.Clock; +import java.time.Duration; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.function.LongSupplier; +import java.util.function.Supplier; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.ActivateMigrationRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentView; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MaintenanceAdmission; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MaintenanceMode; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MetadataMigrationRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MetadataMigrationValidationRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationCapability; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationExportRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationView; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.TargetInspection; +import org.apache.hertzbeat.manager.setup.api.DeploymentWorkflow; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ValidationResponse; +import org.apache.hertzbeat.manager.setup.api.SetupApiException; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.springframework.http.HttpStatus; + +/** Authenticated managed-only deployment workflow over one retained migration graph. */ +public final class DefaultDeploymentWorkflow implements DeploymentWorkflow { + + private final Object migrateAdmission = new Object(); + private final DeploymentViewProjector projector; + private final ManagedDeploymentMigrationCommands commands; + private final MetadataMigrationTargetInspector inspector; + private final MetadataMigrationPolicy policy; + private final DeploymentWorkflowFailureMapper failures; + private final Clock clock; + private final Duration timeout; + private final LongSupplier ticker; + private final boolean available; + private boolean closed; + private int activeCalls; + + DefaultDeploymentWorkflow( + DeploymentViewProjector projector, + ManagedDeploymentMigrationCommands commands, + MetadataMigrationTargetInspector inspector, + MetadataMigrationPolicy policy, + DeploymentWorkflowFailureMapper failures, + Clock clock, + Duration timeout, + LongSupplier ticker) { + this.projector = Objects.requireNonNull(projector, "projector"); + this.commands = Objects.requireNonNull(commands, "commands"); + this.inspector = Objects.requireNonNull(inspector, "inspector"); + this.policy = Objects.requireNonNull(policy, "policy"); + this.failures = Objects.requireNonNull(failures, "failures"); + this.clock = Objects.requireNonNull(clock, "clock"); + this.timeout = requirePositive(timeout); + this.ticker = Objects.requireNonNull(ticker, "ticker"); + available = true; + } + + private DefaultDeploymentWorkflow( + DeploymentViewProjector projector, + DeploymentWorkflowFailureMapper failures, + Clock clock) { + this.projector = Objects.requireNonNull(projector, "projector"); + commands = null; + inspector = null; + policy = null; + this.failures = Objects.requireNonNull(failures, "failures"); + this.clock = Objects.requireNonNull(clock, "clock"); + timeout = Duration.ofSeconds(1); + ticker = System::nanoTime; + available = false; + } + + static DefaultDeploymentWorkflow unavailable( + DeploymentViewProjector projector, + DeploymentWorkflowFailureMapper failures, + Clock clock) { + return new DefaultDeploymentWorkflow(projector, failures, clock); + } + + @Override + public DeploymentView deployment() { + return invoke(false, projector::project); + } + + @Override + public ValidationResponse validate(MetadataMigrationValidationRequest request) { + Objects.requireNonNull(request, "request"); + return invoke(true, () -> validateTarget(request)); + } + + @Override + public MigrationView migrate(MetadataMigrationRequest request) { + Objects.requireNonNull(request, "request"); + synchronized (migrateAdmission) { + return invoke(true, () -> { + requireManaged(request.applyMode()); + return migrateManaged(request); + }); + } + } + + @Override + public MigrationView migration(String operationId) { + return invoke(true, () -> commands.migration(operationId)); + } + + @Override + public MigrationView activate(String operationId, ActivateMigrationRequest request) { + return invoke(true, () -> { + requireOperationalAdmission(operationId); + return commands.activate(operationId, request); + }); + } + + @Override + public PreparedMigrationExport prepareExport( + String operationId, MigrationExportRequest request) { + return invoke(true, () -> { + throw unavailable(); + }); + } + + synchronized void closeAdmission() { + closed = true; + boolean interrupted = Thread.interrupted(); + try { + while (activeCalls != 0) { + try { + wait(); + } catch (InterruptedException waitInterrupted) { + interrupted = true; + } + } + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + + private ValidationResponse validateTarget(MetadataMigrationValidationRequest request) { + DeploymentView deployment = projector.project(); + if (!deployment.migration().allowed()) { + return invalid(deployment.migration().blockedBy()); + } + TargetInspection inspection = inspect(request.targetDatabase()); + return switch (inspection) { + case EMPTY -> new ValidationResponse(true, clock.instant(), null, List.of()); + case NON_EMPTY -> invalid(SetupErrorCode.MIGRATION_TARGET_NOT_EMPTY); + case UNKNOWN -> invalid(SetupErrorCode.METADATA_CONNECTION_FAILED); + }; + } + + private MigrationView migrateManaged(MetadataMigrationRequest request) { + MigrationView existing = existing(request.operationId()); + if (existing != null && existing.state() != MigrationOperationState.PENDING) { + requireTarget(existing, request); + return existing; + } + Optional joined = commands.joinExecuting(request); + if (joined.isPresent()) { + return joined.orElseThrow(); + } + MigrationView settled = existing(request.operationId()); + if (settled != null && settled.state() != MigrationOperationState.PENDING) { + requireTarget(settled, request); + return settled; + } + Optional active = commands.activeOperationId(); + if (active.isPresent() && !active.orElseThrow().equals(request.operationId())) { + throw new MigrationOperationStoreException(SetupErrorCode.OPERATION_CONFLICT); + } + DeploymentView deployment = admission(projector.project(), request.operationId()); + TargetInspection inspection = inspect(request.targetDatabase()); + policy.requireMigrationAllowed(deployment, request.target(), inspection); + return commands.migrate(request); + } + + private MigrationView existing(String operationId) { + try { + return commands.migration(operationId); + } catch (MigrationOperationStoreException failure) { + if (failure.errorCode() == SetupErrorCode.OPERATION_NOT_FOUND) { + return null; + } + throw failure; + } + } + + private TargetInspection inspect(MetadataDatabaseConfiguration database) { + MetadataDatabaseSettings settings = new MetadataDatabaseSettings( + database.kind(), database.jdbcUrl(), database.username()); + try (SecretValue password = SecretValue.of(database.password())) { + JdbcMetadataMigrationDeadline deadline = + JdbcMetadataMigrationDeadline.start(timeout, ticker); + inspector.retryCleanup(deadline); + return inspector.inspect(settings, password, deadline); + } + } + + private void requireOperationalAdmission(String operationId) { + DeploymentView current = admission(projector.project(), operationId); + if (!current.migration().allowed()) { + throw unavailable(); + } + } + + private static DeploymentView admission(DeploymentView deployment, String operationId) { + MigrationCapability migration = deployment.migration(); + if (migration.blockedBy() != SetupErrorCode.OPERATION_CONFLICT + || !operationId.equals(migration.activeOperationId())) { + return deployment; + } + MaintenanceAdmission maintenance = deployment.maintenanceMode() == MaintenanceMode.ACTIVE + ? MaintenanceAdmission.USE_CURRENT : MaintenanceAdmission.AUTO_ENTER; + return new DeploymentView( + deployment.observedAt(), deployment.managementDatabase(), deployment.greptimeDatabase(), + deployment.applyMode(), deployment.maintenanceMode(), deployment.topology(), + MigrationCapability.permitted(maintenance)); + } + + private static void requireTarget(MigrationView existing, MetadataMigrationRequest request) { + if (existing.target() != request.target()) { + throw new MigrationOperationStoreException(SetupErrorCode.OPERATION_CONFLICT); + } + } + + private static void requireManaged(ApplyMode applyMode) { + if (applyMode != ApplyMode.MANAGED_WRITE) { + throw new SetupApiException(SetupErrorCode.INVALID_REQUEST, HttpStatus.BAD_REQUEST); + } + } + + private ValidationResponse invalid(SetupErrorCode code) { + return new ValidationResponse(false, clock.instant(), code, List.of()); + } + + private T translate(Supplier action) { + try { + return action.get(); + } catch (RuntimeException failure) { + throw failures.translate(failure); + } + } + + private T invoke(boolean requiresAvailable, Supplier action) { + begin(requiresAvailable); + try { + return translate(action); + } finally { + end(); + } + } + + private synchronized void begin(boolean requiresAvailable) { + if (closed || requiresAvailable && !available) { + throw unavailable(); + } + activeCalls++; + } + + private synchronized void end() { + activeCalls--; + if (activeCalls == 0) { + notifyAll(); + } + } + + private static SetupApiException unavailable() { + return new SetupApiException( + SetupErrorCode.MIGRATION_UNAVAILABLE, HttpStatus.SERVICE_UNAVAILABLE); + } + + private static Duration requirePositive(Duration timeout) { + Objects.requireNonNull(timeout, "timeout"); + if (timeout.isZero() || timeout.isNegative()) { + throw new IllegalArgumentException("Migration timeout must be positive"); + } + return timeout; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationCommandRunner.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationCommandRunner.java index 376a6c2f83..00338f5c42 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationCommandRunner.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationCommandRunner.java @@ -87,6 +87,24 @@ final class DeploymentMigrationCommandRunner implements AutoCloseable { return barrier.await(timeout); } + Optional joinExecuting(MetadataMigrationRequest request) { + Objects.requireNonNull(request, "request"); + requireManaged(request.applyMode()); + MigrationTargetRequest target = taskFactory.request(request); + MigrationPreparationBarrier barrier; + synchronized (this) { + requireOpen(); + if (active == null || !active.executing()) { + return Optional.empty(); + } + if (!active.operationId().equals(target.operationId()) || !active.matches(target)) { + throw failure(SetupErrorCode.OPERATION_CONFLICT); + } + barrier = active.barrier(); + } + return Optional.of(barrier.await(timeout)); + } + Optional find(String operationId) { Optional persisted = store.find(operationId); synchronized (this) { @@ -98,6 +116,13 @@ final class DeploymentMigrationCommandRunner implements AutoCloseable { return persisted.map(MigrationOperationProjection::view); } + synchronized Optional inFlightSnapshot(String operationId) { + if (active == null || !active.operationId().equals(operationId) || !active.executing()) { + return Optional.empty(); + } + return active.barrier().confirmedSnapshot(); + } + synchronized Optional activeOperationId() { if (active != null) { return Optional.of(active.operationId()); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationConfiguration.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationConfiguration.java new file mode 100644 index 0000000000..318832c519 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationConfiguration.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.nio.file.Path; +import java.time.Clock; +import java.time.Duration; +import org.apache.hertzbeat.common.runtime.BusinessRuntimeGate; +import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceCoordinator; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceOrchestrator; +import org.apache.hertzbeat.manager.maintenance.StandaloneDeploymentOwnerView; +import org.apache.hertzbeat.manager.setup.api.DeploymentWorkflow; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; +import org.apache.hertzbeat.manager.setup.config.SetupInstallationPaths; +import org.springframework.beans.factory.FactoryBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; + +/** NORMAL-only construction boundary for the single managed migration runtime graph. */ +@Configuration(proxyBeanMethods = false) +@ConditionalOnNormalBusinessRuntime +@ConditionalOnBean(StandaloneDeploymentOwnerView.class) +public class DeploymentMigrationConfiguration { + + private static final Duration OPERATION_TIMEOUT = Duration.ofMinutes(5); + + @Bean + @ConditionalOnMissingBean(DeploymentMigrationRuntime.Opener.class) + DeploymentMigrationRuntime.Opener deploymentMigrationRuntimeOpener() { + return DeploymentMigrationRuntime::open; + } + + @Bean(destroyMethod = "destroySafely") + DeploymentMigrationRuntime deploymentMigrationRuntime( + Environment environment, + BusinessRuntimeGate runtimeGate, + ManagedConfigCapability capability, + StandaloneDeploymentOwnerView owner, + SetupRuntimeState state, + MetadataMaintenanceCoordinator maintenance, + MigrationMaintenanceOrchestrator maintenanceOrchestrator, + DeploymentMigrationRuntime.Opener opener) { + Path root = SetupInstallationPaths.root(environment); + Clock clock = Clock.systemUTC(); + if (!admitted(runtimeGate, capability, owner, root)) { + DeploymentViewProjector projector = DeploymentViewProjector.unavailable( + state, capability, owner, maintenance, clock); + DefaultDeploymentWorkflow unavailable = DefaultDeploymentWorkflow.unavailable( + projector, new DeploymentWorkflowFailureMapper(), clock); + return DeploymentMigrationRuntime.unavailable(unavailable); + } + return opener.open(new DeploymentMigrationRuntime.OpenContext( + root, owner, state, capability, maintenance, maintenanceOrchestrator, + clock, OPERATION_TIMEOUT, System::nanoTime)); + } + + @Bean + FactoryBean deploymentWorkflow(DeploymentMigrationRuntime runtime) { + return new DeploymentWorkflowFactory(runtime); + } + + private static boolean admitted( + BusinessRuntimeGate gate, + ManagedConfigCapability capability, + StandaloneDeploymentOwnerView owner, + Path root) { + if (!gate.isOpen() + || !capability.writableManagedConfig() + || capability.applyMode() != ApplyMode.MANAGED_WRITE + || !owner.isValid()) { + return false; + } + return root.equals(owner.installationRoot().toAbsolutePath().normalize()); + } + + private static final class DeploymentWorkflowFactory implements FactoryBean { + + private final DeploymentMigrationRuntime runtime; + + private DeploymentWorkflowFactory(DeploymentMigrationRuntime runtime) { + this.runtime = runtime; + } + + @Override + public DeploymentWorkflow getObject() { + return runtime.available() ? runtime.workflow() : null; + } + + @Override + public Class getObjectType() { + return runtime.available() ? DeploymentWorkflow.class : null; + } + + @Override + public boolean isSingleton() { + return true; + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationRuntime.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationRuntime.java new file mode 100644 index 0000000000..d90a72146e --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationRuntime.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.nio.file.Path; +import java.time.Clock; +import java.time.Duration; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.LockSupport; +import java.util.function.LongSupplier; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceCoordinator; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceOrchestrator; +import org.apache.hertzbeat.manager.maintenance.StandaloneDeploymentOwnerView; +import org.apache.hertzbeat.manager.setup.api.DeploymentWorkflow; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction; + +/** Owns the one process-local migration graph and closes its exact owners in safety order. */ +final class DeploymentMigrationRuntime implements AutoCloseable { + + private static final long INITIAL_DESTROY_BACKOFF_NANOS = TimeUnit.MILLISECONDS.toNanos(1); + private static final long MAX_DESTROY_BACKOFF_NANOS = TimeUnit.MILLISECONDS.toNanos(100); + + private final DefaultDeploymentWorkflow workflow; + private final ManagedDeploymentMigrationCommands commands; + private final MetadataMigrationTargetInspector inspector; + private final JdbcMetadataMigrationExecutor copyExecutor; + private final TargetJdbcConnectionFactory factory; + private final Duration cleanupTimeout; + private final LongSupplier ticker; + private ClosePhase closePhase = ClosePhase.ADMISSION; + + DeploymentMigrationRuntime( + DefaultDeploymentWorkflow workflow, + ManagedDeploymentMigrationCommands commands, + MetadataMigrationTargetInspector inspector, + JdbcMetadataMigrationExecutor copyExecutor, + TargetJdbcConnectionFactory factory, + Duration cleanupTimeout, + LongSupplier ticker) { + this.workflow = Objects.requireNonNull(workflow, "workflow"); + this.commands = Objects.requireNonNull(commands, "commands"); + this.inspector = Objects.requireNonNull(inspector, "inspector"); + this.copyExecutor = Objects.requireNonNull(copyExecutor, "copyExecutor"); + this.factory = Objects.requireNonNull(factory, "factory"); + this.cleanupTimeout = requirePositive(cleanupTimeout); + this.ticker = Objects.requireNonNull(ticker, "ticker"); + } + + private DeploymentMigrationRuntime(DefaultDeploymentWorkflow workflow) { + this.workflow = Objects.requireNonNull(workflow, "workflow"); + commands = null; + inspector = null; + copyExecutor = null; + factory = null; + cleanupTimeout = Duration.ofSeconds(1); + ticker = System::nanoTime; + closePhase = ClosePhase.CLOSED; + } + + static DeploymentMigrationRuntime unavailable(DefaultDeploymentWorkflow workflow) { + return new DeploymentMigrationRuntime(workflow); + } + + static DeploymentMigrationRuntime open(OpenContext context) { + return open(context, stage -> { }); + } + + static DeploymentMigrationRuntime open( + OpenContext context, ConstructionCheckpoint checkpoint) { + Objects.requireNonNull(context, "context").validate(); + Objects.requireNonNull(checkpoint, "checkpoint"); + TargetJdbcConnectionFactory factory = new TargetJdbcConnectionFactory( + TargetJdbcAbortExecutor.instance()); + MetadataMigrationTargetInspector inspector = new MetadataMigrationTargetInspector( + factory, new TargetSchemaReadOnlyInspector()); + JdbcMetadataMigrationExecutor copyExecutor = new JdbcMetadataMigrationExecutor(); + DeploymentMigrationCommandRunner runner = null; + ManagedDeploymentMigrationCommands commands = null; + try { + FileMigrationOperationStore store = new FileMigrationOperationStore(context.root()); + ManagedMigrationConfigurationTransaction configuration = + new ManagedMigrationConfigurationTransaction(context.root()); + RetainedCutoverCoordinator coordinator = new RetainedCutoverCoordinator( + factory, new FlywayTargetSchemaProvisioner(), context.maintenanceOrchestrator(), + copyExecutor, context.ticker()); + runner = new DeploymentMigrationCommandRunner( + store, configuration, coordinator, context.clock(), context.timeout()); + checkpoint.reached(ConstructionStage.RUNNER); + commands = new ManagedDeploymentMigrationCommands( + runner, store, configuration, coordinator); + checkpoint.reached(ConstructionStage.COMMANDS); + DeploymentViewProjector projector = new DeploymentViewProjector( + context.state(), context.capability(), context.owner(), context.maintenance(), + commands, context.clock()); + DefaultDeploymentWorkflow workflow = new DefaultDeploymentWorkflow( + projector, commands, inspector, new MetadataMigrationPolicy(), + new DeploymentWorkflowFailureMapper(), context.clock(), context.timeout(), + context.ticker()); + checkpoint.reached(ConstructionStage.WORKFLOW); + return new DeploymentMigrationRuntime( + workflow, commands, inspector, copyExecutor, factory, + context.timeout(), context.ticker()); + } catch (RuntimeException | Error failure) { + Cleanup commandCleanup = commands == null + ? runner == null ? null : runner::close + : commands::close; + Throwable result = closeConstruction( + commandCleanup, inspector, copyExecutor, factory, context, failure); + if (result instanceof Error fatal) { + throw fatal; + } + throw (RuntimeException) result; + } + } + + DeploymentWorkflow workflow() { + return workflow; + } + + boolean available() { + return commands != null; + } + + @Override + public synchronized void close() { + while (closePhase != ClosePhase.CLOSED) { + switch (closePhase) { + case ADMISSION -> closeStep(workflow::closeAdmission, ClosePhase.COMMANDS); + case COMMANDS -> closeStep(commands::close, ClosePhase.INSPECTOR); + case INSPECTOR -> closeStep( + () -> inspector.shutdown(deadline()), ClosePhase.COPY_EXECUTOR); + case COPY_EXECUTOR -> closeStep(copyExecutor::close, ClosePhase.FACTORY); + case FACTORY -> closeStep(factory::close, ClosePhase.CLOSED); + case CLOSED -> throw new IllegalStateException("Unexpected closed phase"); + default -> throw new IllegalStateException("Unknown close phase"); + } + } + } + + synchronized void destroySafely() { + boolean interrupted = Thread.interrupted(); + JdbcMetadataMigrationDeadline retryBudget = deadline(); + long backoff = INITIAL_DESTROY_BACKOFF_NANOS; + try { + while (closePhase != ClosePhase.CLOSED) { + try { + close(); + } catch (RuntimeException ignored) { + interrupted |= Thread.interrupted(); + long remaining = retryBudget.remainingNanos(); + long pause = remaining > 0 + ? Math.min(backoff, remaining) : MAX_DESTROY_BACKOFF_NANOS; + LockSupport.parkNanos(pause); + interrupted |= Thread.interrupted(); + backoff = Math.min(MAX_DESTROY_BACKOFF_NANOS, backoff * 2); + } + } + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + + private void closeStep(Cleanup cleanup, ClosePhase next) { + runInterruptIsolated(cleanup); + closePhase = next; + } + + private JdbcMetadataMigrationDeadline deadline() { + return JdbcMetadataMigrationDeadline.start(cleanupTimeout, ticker); + } + + private static Throwable closeConstruction( + Cleanup commandCleanup, + MetadataMigrationTargetInspector inspector, + JdbcMetadataMigrationExecutor executor, + TargetJdbcConnectionFactory factory, + OpenContext context, + Throwable primary) { + Throwable result = primary; + if (commandCleanup != null) { + result = runCleanupAfter(result, commandCleanup); + } + result = runCleanupAfter(result, () -> inspector.shutdown( + JdbcMetadataMigrationDeadline.start(context.timeout(), context.ticker()))); + result = runCleanupAfter(result, executor::close); + return runCleanupAfter(result, factory::close); + } + + static Throwable runCleanupAfter(Throwable primary, Cleanup cleanup) { + try { + runInterruptIsolated(cleanup); + return primary; + } catch (Error cleanupFatal) { + if (primary instanceof Error) { + primary.addSuppressed(recoveryMarker()); + return primary; + } + cleanupFatal.addSuppressed(recoveryMarker()); + return cleanupFatal; + } catch (RuntimeException ignored) { + return primary; + } + } + + private static MigrationOperationStoreException recoveryMarker() { + return new MigrationOperationStoreException( + org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode + .CONFIG_RECOVERY_REQUIRED); + } + + private static void runInterruptIsolated(Cleanup cleanup) { + boolean interrupted = Thread.interrupted(); + try { + cleanup.run(); + } finally { + interrupted |= Thread.interrupted(); + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + + private static Duration requirePositive(Duration timeout) { + Objects.requireNonNull(timeout, "timeout"); + if (timeout.isNegative() || timeout.isZero()) { + throw new IllegalArgumentException("Migration timeout must be positive"); + } + return timeout; + } + + @FunctionalInterface + interface Opener { + DeploymentMigrationRuntime open(OpenContext context); + } + + @FunctionalInterface + interface ConstructionCheckpoint { + void reached(ConstructionStage stage); + } + + enum ConstructionStage { + RUNNER, + COMMANDS, + WORKFLOW + } + + record OpenContext( + Path root, + StandaloneDeploymentOwnerView owner, + SetupRuntimeState state, + ManagedConfigCapability capability, + MetadataMaintenanceCoordinator maintenance, + MigrationMaintenanceOrchestrator maintenanceOrchestrator, + Clock clock, + Duration timeout, + LongSupplier ticker) { + + OpenContext { + root = Objects.requireNonNull(root, "root").toAbsolutePath().normalize(); + Objects.requireNonNull(owner, "owner"); + Objects.requireNonNull(state, "state"); + Objects.requireNonNull(capability, "capability"); + Objects.requireNonNull(maintenance, "maintenance"); + Objects.requireNonNull(maintenanceOrchestrator, "maintenanceOrchestrator"); + Objects.requireNonNull(clock, "clock"); + requirePositive(timeout); + Objects.requireNonNull(ticker, "ticker"); + } + + void validate() { + if (!capability.writableManagedConfig() + || !owner.isValid() + || !root.equals(owner.installationRoot().toAbsolutePath().normalize())) { + throw new IllegalArgumentException("Migration runtime admission is unavailable"); + } + } + } + + @FunctionalInterface + interface Cleanup { + void run(); + } + + private enum ClosePhase { + ADMISSION, + COMMANDS, + INSPECTOR, + COPY_EXECUTOR, + FACTORY, + CLOSED + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentViewProjector.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentViewProjector.java new file mode 100644 index 0000000000..966db7aad2 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentViewProjector.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.time.Clock; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Supplier; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceCoordinator; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenancePhase; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceSnapshot; +import org.apache.hertzbeat.manager.maintenance.StandaloneDeploymentOwnerView; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentTopology; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentView; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MaintenanceAdmission; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MaintenanceMode; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationCapability; +import org.apache.hertzbeat.manager.setup.api.OperationIdValidator; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.StatusResponse; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; + +/** Builds the secret-free deployment view from authoritative in-process facts. */ +final class DeploymentViewProjector { + + private final SetupRuntimeState state; + private final ManagedConfigCapability capability; + private final StandaloneDeploymentOwnerView owner; + private final MetadataMaintenanceCoordinator maintenance; + private final Supplier> activeOperation; + private final Clock clock; + private final boolean forceUnavailable; + + DeploymentViewProjector( + SetupRuntimeState state, + ManagedConfigCapability capability, + StandaloneDeploymentOwnerView owner, + MetadataMaintenanceCoordinator maintenance, + ManagedDeploymentMigrationCommands commands, + Clock clock) { + this(state, capability, owner, maintenance, commands::activeOperationId, clock); + } + + DeploymentViewProjector( + SetupRuntimeState state, + ManagedConfigCapability capability, + StandaloneDeploymentOwnerView owner, + MetadataMaintenanceCoordinator maintenance, + Supplier> activeOperation, + Clock clock) { + this(state, capability, owner, maintenance, activeOperation, clock, false); + } + + private DeploymentViewProjector( + SetupRuntimeState state, + ManagedConfigCapability capability, + StandaloneDeploymentOwnerView owner, + MetadataMaintenanceCoordinator maintenance, + Supplier> activeOperation, + Clock clock, + boolean forceUnavailable) { + this.state = Objects.requireNonNull(state, "state"); + this.capability = Objects.requireNonNull(capability, "capability"); + this.owner = Objects.requireNonNull(owner, "owner"); + this.maintenance = Objects.requireNonNull(maintenance, "maintenance"); + this.activeOperation = Objects.requireNonNull(activeOperation, "activeOperation"); + this.clock = Objects.requireNonNull(clock, "clock"); + this.forceUnavailable = forceUnavailable; + } + + static DeploymentViewProjector unavailable( + SetupRuntimeState state, + ManagedConfigCapability capability, + StandaloneDeploymentOwnerView owner, + MetadataMaintenanceCoordinator maintenance, + Clock clock) { + return new DeploymentViewProjector( + state, capability, owner, maintenance, Optional::empty, clock, true); + } + + DeploymentView project() { + StatusResponse status = state.status(); + if (forceUnavailable) { + DeploymentTopology topology = owner.isValid() + ? DeploymentTopology.SINGLE_NODE : DeploymentTopology.UNKNOWN; + return new DeploymentView( + clock.instant(), status.managementDatabase(), status.telemetryStore(), + capability.applyMode(), MaintenanceMode.INACTIVE, topology, unavailable()); + } + MetadataMaintenanceSnapshot maintenanceSnapshot = maintenance.snapshot(); + MaintenanceMode maintenanceMode = maintenanceSnapshot.phase() == MetadataMaintenancePhase.RUNNING + ? MaintenanceMode.INACTIVE : MaintenanceMode.ACTIVE; + DeploymentTopology topology = owner.isValid() + ? DeploymentTopology.SINGLE_NODE : DeploymentTopology.UNKNOWN; + Optional operationId = activeOperation.get(); + MigrationCapability migration = migration( + status.managementDatabase().kind(), topology, maintenanceSnapshot, operationId); + return new DeploymentView( + clock.instant(), status.managementDatabase(), status.telemetryStore(), + capability.applyMode(), maintenanceMode, topology, migration); + } + + private MigrationCapability migration( + MetadataDatabaseKind database, + DeploymentTopology topology, + MetadataMaintenanceSnapshot snapshot, + Optional active) { + if (database != MetadataDatabaseKind.H2) { + return structural(SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED); + } + if (topology != DeploymentTopology.SINGLE_NODE) { + return structural(SetupErrorCode.MIGRATION_TOPOLOGY_UNAVAILABLE); + } + if (!capability.writableManagedConfig() + || capability.applyMode() != ApplyMode.MANAGED_WRITE) { + return unavailable(); + } + if (snapshot.phase() == MetadataMaintenancePhase.QUIESCING + || snapshot.phase() == MetadataMaintenancePhase.RECOVERY_REQUIRED) { + return unavailable(); + } + if (active.isPresent()) { + return conflict(active.orElseThrow()); + } + if (snapshot.operationId() != null) { + return OperationIdValidator.isSafe(snapshot.operationId()) + ? conflict(snapshot.operationId()) : unavailable(); + } + return switch (snapshot.phase()) { + case RUNNING -> MigrationCapability.permitted(MaintenanceAdmission.AUTO_ENTER); + case QUIESCED -> MigrationCapability.permitted(MaintenanceAdmission.USE_CURRENT); + case QUIESCING, RECOVERY_REQUIRED -> unavailable(); + }; + } + + private static MigrationCapability structural(SetupErrorCode blocker) { + return MigrationCapability.blocked(blocker, MaintenanceAdmission.NOT_APPLICABLE); + } + + private static MigrationCapability unavailable() { + return MigrationCapability.blocked( + SetupErrorCode.MIGRATION_UNAVAILABLE, MaintenanceAdmission.UNAVAILABLE); + } + + private static MigrationCapability conflict(String operationId) { + return MigrationCapability.blocked( + SetupErrorCode.OPERATION_CONFLICT, MaintenanceAdmission.UNAVAILABLE, operationId); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentWorkflowFailureMapper.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentWorkflowFailureMapper.java new file mode 100644 index 0000000000..693aef266d --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentWorkflowFailureMapper.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceErrorCode; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceException; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiException; +import org.springframework.http.HttpStatus; + +/** Converts only known secret-free failures into the stable deployment HTTP contract. */ +final class DeploymentWorkflowFailureMapper { + + RuntimeException translate(RuntimeException failure) { + if (failure instanceof SetupApiException) { + return failure; + } + SetupErrorCode code = stableCode(failure); + return new SetupApiException( + code == null ? SetupErrorCode.MIGRATION_UNAVAILABLE : code, + status(code == null ? SetupErrorCode.MIGRATION_UNAVAILABLE : code)); + } + + private static SetupErrorCode stableCode(RuntimeException failure) { + if (failure instanceof MigrationOperationStoreException storeFailure) { + return storeFailure.errorCode(); + } + if (failure instanceof DurableCutoverPreparationException preparationFailure) { + return preparationFailure.errorCode(); + } + if (failure instanceof RetainedCopyJournalHandoffException handoffFailure) { + return handoffFailure.errorCode(); + } + if (failure instanceof RetainedManagedActivationException activationFailure) { + return activationFailure.errorCode(); + } + if (failure instanceof TargetJdbcConnectionException connectionFailure) { + return connectionCode(connectionFailure.code()); + } + if (failure instanceof MigrationMaintenanceException maintenanceFailure) { + return maintenanceCode(maintenanceFailure.code()); + } + if (failure instanceof MetadataMigrationException migrationFailure) { + return migrationCode(migrationFailure.code()); + } + if (failure instanceof RetainedCutoverReleaseRequiredException + || failure instanceof MetadataCopyReleaseRequiredException) { + return SetupErrorCode.CONFIG_RECOVERY_REQUIRED; + } + if (failure instanceof RetainedCutoverException cutoverFailure) { + return switch (cutoverFailure.code()) { + case TARGET_IDENTITY_CHANGED -> SetupErrorCode.OPERATION_CONFLICT; + case PREPARATION_RETRY_REQUIRED -> SetupErrorCode.CONFIG_RECOVERY_REQUIRED; + case EXECUTION_FAILED -> SetupErrorCode.MIGRATION_UNAVAILABLE; + }; + } + return null; + } + + private static SetupErrorCode migrationCode(MetadataMigrationErrorCode code) { + return switch (code) { + case SCHEMA, COPY -> SetupErrorCode.MIGRATION_COPY_FAILED; + case VERIFICATION, SEQUENCE -> SetupErrorCode.MIGRATION_VERIFICATION_FAILED; + case TIMEOUT -> SetupErrorCode.MIGRATION_UNAVAILABLE; + case COMMIT_OUTCOME_UNKNOWN, ROLLBACK_OUTCOME_UNKNOWN -> + SetupErrorCode.CONFIG_RECOVERY_REQUIRED; + }; + } + + private static SetupErrorCode connectionCode(TargetJdbcConnectionErrorCode code) { + return switch (code) { + case TARGET_MISMATCH -> SetupErrorCode.METADATA_SCHEMA_MISMATCH; + case OPERATION_CONFLICT -> SetupErrorCode.OPERATION_CONFLICT; + case TIMEOUT, UNAVAILABLE, FACTORY_CLOSED, CLEANUP_REQUIRED -> + SetupErrorCode.MIGRATION_UNAVAILABLE; + }; + } + + private static SetupErrorCode maintenanceCode(MigrationMaintenanceErrorCode code) { + return switch (code) { + case INVALID_REQUEST -> SetupErrorCode.INVALID_REQUEST; + case MIGRATION_MULTI_NODE_UNSUPPORTED -> SetupErrorCode.MIGRATION_MULTI_NODE_UNSUPPORTED; + case MIGRATION_OPERATION_CONFLICT -> SetupErrorCode.OPERATION_CONFLICT; + case MIGRATION_DEPLOYMENT_AUTHORITY_UNAVAILABLE, + MIGRATION_SOURCE_UNAVAILABLE, + MIGRATION_MAINTENANCE_TIMEOUT, + MIGRATION_MAINTENANCE_INTERRUPTED, + MIGRATION_MAINTENANCE_FAILURE, + MIGRATION_RESUME_FAILURE -> SetupErrorCode.MIGRATION_UNAVAILABLE; + }; + } + + private static HttpStatus status(SetupErrorCode code) { + return switch (code) { + case INVALID_REQUEST -> HttpStatus.BAD_REQUEST; + case OPERATION_NOT_FOUND -> HttpStatus.NOT_FOUND; + case OPERATION_CONFLICT, + MIGRATION_SOURCE_UNSUPPORTED, + MIGRATION_TARGET_NOT_EMPTY, + MIGRATION_MULTI_NODE_UNSUPPORTED, + METADATA_CONNECTION_FAILED, + METADATA_SCHEMA_MISMATCH, + MIGRATION_ACTIVATION_NOT_AVAILABLE, + MIGRATION_COPY_FAILED, + MIGRATION_VERIFICATION_FAILED, + MIGRATION_ACTIVATION_FAILED, + RESTART_FAILED -> HttpStatus.CONFLICT; + default -> HttpStatus.SERVICE_UNAVAILABLE; + }; + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java index 492d486faf..094f65a857 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/FileMigrationOperationStore.java @@ -22,6 +22,7 @@ import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; import org.apache.hertzbeat.manager.setup.security.CommittedSetupFileDurabilityException; import org.apache.hertzbeat.manager.setup.security.SecureSetupFile; import org.apache.hertzbeat.manager.setup.security.SecureSetupFileLock; +import org.apache.hertzbeat.manager.setup.security.SecureSetupFileLock.TryResult; /** Root-bound owner-only file adapter for the single active migration operation. */ public final class FileMigrationOperationStore implements MigrationOperationStore { @@ -100,6 +101,21 @@ public final class FileMigrationOperationStore implements MigrationOperationStor .filter(snapshot -> snapshot.operationId().equals(operationId)).findFirst()); } + NonblockingSnapshotRead tryFind(String operationId) { + requireSafeId(operationId); + try { + TryResult> result = lock.tryExecute(() -> read().stream() + .filter(snapshot -> snapshot.operationId().equals(operationId)).findFirst()); + if (!result.acquired()) { + return NonblockingSnapshotRead.busy(); + } + return result.value().map(NonblockingSnapshotRead::present) + .orElseGet(NonblockingSnapshotRead::missing); + } catch (IOException | RuntimeException ignored) { + throw failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + } + /** Selects one startup record only when no other operation still owns migration progress. */ Optional selectForStartup(String operationId) { requireSafeId(operationId); @@ -360,4 +376,21 @@ public final class FileMigrationOperationStore implements MigrationOperationStor } enum ExactTransitionDisposition { TRANSITIONED, ALREADY_CONFIRMED } + + record NonblockingSnapshotRead(ReadState state, MigrationOperationSnapshot snapshot) { + + static NonblockingSnapshotRead busy() { + return new NonblockingSnapshotRead(ReadState.BUSY, null); + } + + static NonblockingSnapshotRead missing() { + return new NonblockingSnapshotRead(ReadState.MISSING, null); + } + + static NonblockingSnapshotRead present(MigrationOperationSnapshot snapshot) { + return new NonblockingSnapshotRead(ReadState.PRESENT, snapshot); + } + } + + enum ReadState { BUSY, MISSING, PRESENT } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedDeploymentMigrationCommands.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedDeploymentMigrationCommands.java index 5eb16a073f..0c2a5616e6 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedDeploymentMigrationCommands.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/ManagedDeploymentMigrationCommands.java @@ -50,8 +50,19 @@ final class ManagedDeploymentMigrationCommands implements AutoCloseable { MigrationView migration(String operationId) { requireOpen(); requireOperationId(operationId); - Optional stored = runner.find(operationId); RetainedCutoverStatus retained = coordinator.status(); + if (retained.owns(operationId) && liveCopyPhase(retained.phase())) { + Optional proof = runner.inFlightSnapshot(operationId); + if (proof.isPresent()) { + return liveProjection(operationId, proof.orElseThrow()); + } + retained = coordinator.status(); + } + Optional stored = retained.owns(operationId) + && confirmedRetainedPhase(retained.phase()) + ? store.find(operationId).map(MigrationOperationProjection::view) + : runner.find(operationId); + retained = coordinator.status(); if (retained.owns(operationId) && !matchesRetainedShape(stored, retained.phase())) { throw failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); } @@ -93,6 +104,12 @@ final class ManagedDeploymentMigrationCommands implements AutoCloseable { return Optional.of(retained.operationId()); } + Optional joinExecuting(MetadataMigrationRequest request) { + requireOpen(); + Objects.requireNonNull(request, "request"); + return runner.joinExecuting(request); + } + @Override public synchronized void close() { closed = true; @@ -140,10 +157,70 @@ final class ManagedDeploymentMigrationCommands implements AutoCloseable { case RETAINED -> view.state() == MigrationOperationState.READY_TO_ACTIVATE; case AWAITING_RESTART_RETAINED -> view.state() == MigrationOperationState.AWAITING_RESTART; + case EXECUTING, HANDOFFING -> + view.state() == MigrationOperationState.RUNNING; default -> false; }).isPresent(); } + private static boolean liveCopyPhase(RetainedCutoverStatus.Phase phase) { + return phase == RetainedCutoverStatus.Phase.EXECUTING + || phase == RetainedCutoverStatus.Phase.HANDOFFING; + } + + private static boolean confirmedRetainedPhase(RetainedCutoverStatus.Phase phase) { + return phase == RetainedCutoverStatus.Phase.RETAINED + || phase == RetainedCutoverStatus.Phase.AWAITING_RESTART_RETAINED; + } + + private MigrationView liveProjection( + String operationId, MigrationOperationSnapshot proof) { + FileMigrationOperationStore.NonblockingSnapshotRead read = store.tryFind(operationId); + return switch (read.state()) { + case BUSY -> MigrationOperationProjection.view(proof); + case MISSING -> throw failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + case PRESENT -> { + MigrationOperationSnapshot current = read.snapshot(); + if (!compatibleLiveSnapshot(proof, current)) { + throw failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + yield current.state() == MigrationOperationState.READY_TO_ACTIVATE + ? finalLiveProjection(proof, current) + : MigrationOperationProjection.view(current); + } + }; + } + + private MigrationView finalLiveProjection( + MigrationOperationSnapshot proof, MigrationOperationSnapshot current) { + RetainedCutoverStatus retained = coordinator.status(); + if (retained.owns(current.operationId()) + && retained.phase() == RetainedCutoverStatus.Phase.RETAINED) { + return MigrationOperationProjection.view(current); + } + if (retained.owns(current.operationId()) && liveCopyPhase(retained.phase())) { + return MigrationOperationProjection.view(proof); + } + throw failure(SetupErrorCode.CONFIG_RECOVERY_REQUIRED); + } + + private static boolean compatibleLiveSnapshot( + MigrationOperationSnapshot proof, MigrationOperationSnapshot current) { + boolean compatibleState = proof.state() == MigrationOperationState.PENDING + ? current.equals(proof) + : current.state() == MigrationOperationState.RUNNING + || current.state() == MigrationOperationState.READY_TO_ACTIVATE; + return compatibleState + && current.operationId().equals(proof.operationId()) + && current.target() == proof.target() + && current.applyMode() == proof.applyMode() + && current.createdAt().equals(proof.createdAt()) + && Objects.equals(current.startedAt(), proof.startedAt()) + && current.targetIdentityHash().equals(proof.targetIdentityHash()) + && Objects.equals(current.managedCandidateGeneration(), + proof.managedCandidateGeneration()); + } + private static void requireOwned(RetainedCutoverStatus retained, String operationId) { if (!retained.owns(operationId)) { throw failure(SetupErrorCode.OPERATION_CONFLICT); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationPreparationBarrier.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationPreparationBarrier.java index dfb4676d29..04c6c8b7bd 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationPreparationBarrier.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/MigrationPreparationBarrier.java @@ -9,6 +9,7 @@ package org.apache.hertzbeat.manager.setup.workflow; import java.time.Duration; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -25,6 +26,7 @@ final class MigrationPreparationBarrier implements RetainedCutoverPreparation { private final CompletableFuture prepared = new CompletableFuture<>(); private DurableCutoverDraft draft; private RetainedCutoverPreparation delegate; + private MigrationOperationSnapshot confirmedSnapshot; MigrationPreparationBarrier(FileMigrationOperationStore store) { this.store = Objects.requireNonNull(store, "store"); @@ -92,6 +94,13 @@ final class MigrationPreparationBarrier implements RetainedCutoverPreparation { } } + synchronized Optional confirmedSnapshot() { + if (!prepared.isDone() || prepared.isCompletedExceptionally()) { + return Optional.empty(); + } + return Optional.ofNullable(confirmedSnapshot); + } + private static long timeoutNanos(Duration timeout) { try { return Math.max(1, timeout.toNanos()); @@ -120,7 +129,11 @@ final class MigrationPreparationBarrier implements RetainedCutoverPreparation { } private void publish(RetainedCutoverPreparationContext context) { - prepared.complete(MigrationOperationProjection.view(requireExactSnapshot(context))); + MigrationOperationSnapshot snapshot = requireExactSnapshot(context); + synchronized (this) { + confirmedSnapshot = snapshot; + } + prepared.complete(MigrationOperationProjection.view(snapshot)); } private MigrationOperationSnapshot requireExactSnapshot( diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcAbortExecutor.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcAbortExecutor.java new file mode 100644 index 0000000000..f432d1f6b5 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/setup/workflow/TargetJdbcAbortExecutor.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.util.concurrent.Executor; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** Process-lifetime bounded daemon executor for JDBC abort calls that may outlive a runtime. */ +final class TargetJdbcAbortExecutor { + + private static final Executor INSTANCE = new ThreadPoolExecutor( + 0, 2, 30, TimeUnit.SECONDS, new SynchronousQueue<>(), runnable -> { + Thread thread = new Thread(runnable, "target-jdbc-abort"); + thread.setDaemon(true); + return thread; + }, new ThreadPoolExecutor.AbortPolicy()); + + private TargetJdbcAbortExecutor() { + } + + static Executor instance() { + return INSTANCE; + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultDeploymentWorkflowTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultDeploymentWorkflowTest.java new file mode 100644 index 0000000000..803ee31d51 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DefaultDeploymentWorkflowTest.java @@ -0,0 +1,421 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BooleanSupplier; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.ActivateMigrationRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentTopology; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentView; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MaintenanceAdmission; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MaintenanceMode; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MetadataMigrationRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MetadataMigrationValidationRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationCapability; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationView; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.TargetInspection; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigSource; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ManagementDatabaseSummary; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreSummary; +import org.apache.hertzbeat.manager.setup.api.SetupApiException; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.mockito.InOrder; + +class DefaultDeploymentWorkflowTest { + + private static final String OPERATION = "operation-a"; + private static final Instant NOW = Instant.parse("2026-08-10T02:00:00Z"); + private static final Duration TIMEOUT = Duration.ofSeconds(5); + + @Test + void durableSameOperationReplayHappensBeforeInspectionAndDoesNotCopySecret() { + Fixture fixture = fixture(); + MigrationView running = MigrationTestSnapshots.running(OPERATION); + when(fixture.commands.migration(OPERATION)).thenReturn(running); + + assertThat(fixture.workflow.migrate(request(OPERATION, ApplyMode.MANAGED_WRITE))) + .isSameAs(running); + + verify(fixture.inspector, never()).inspect(any(), any(), any()); + verify(fixture.commands, never()).migrate(any()); + } + + @Test + @Timeout(10) + void concurrentSameOperationInspectsAndSubmitsOnlyOnceThenReplays() throws Exception { + Fixture fixture = fixture(); + MigrationView running = MigrationTestSnapshots.running(OPERATION); + when(fixture.commands.migration(OPERATION)) + .thenThrow(new MigrationOperationStoreException(SetupErrorCode.OPERATION_NOT_FOUND)) + .thenThrow(new MigrationOperationStoreException(SetupErrorCode.OPERATION_NOT_FOUND)) + .thenReturn(running); + when(fixture.commands.activeOperationId()).thenReturn(Optional.empty()); + when(fixture.inspector.inspect(any(), any(), any())).thenReturn(TargetInspection.EMPTY); + when(fixture.commands.migrate(any())).thenReturn(running); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + when(fixture.inspector.inspect(any(), any(), any())).thenAnswer(invocation -> { + entered.countDown(); + assertThat(release.await(5, TimeUnit.SECONDS)).isTrue(); + return TargetInspection.EMPTY; + }); + + try (var callers = Executors.newVirtualThreadPerTaskExecutor()) { + Future first = callers.submit( + () -> fixture.workflow.migrate(request(OPERATION, ApplyMode.MANAGED_WRITE))); + assertThat(entered.await(5, TimeUnit.SECONDS)).isTrue(); + Future second = callers.submit( + () -> fixture.workflow.migrate(request(OPERATION, ApplyMode.MANAGED_WRITE))); + release.countDown(); + + assertThat(first.get(5, TimeUnit.SECONDS)).isSameAs(running); + assertThat(second.get(5, TimeUnit.SECONDS)).isSameAs(running); + } finally { + release.countDown(); + } + verify(fixture.inspector).inspect(any(), any(), any()); + verify(fixture.commands).migrate(any()); + verify(fixture.commands, times(3)).migration(OPERATION); + } + + @Test + void foreignOperationStopsTargetInspection() { + Fixture fixture = fixture(); + when(fixture.commands.migration(OPERATION)) + .thenThrow(new MigrationOperationStoreException(SetupErrorCode.OPERATION_NOT_FOUND)); + when(fixture.commands.activeOperationId()).thenReturn(Optional.of("operation-b")); + + assertThatThrownBy(() -> fixture.workflow.migrate(request(OPERATION, ApplyMode.MANAGED_WRITE))) + .isInstanceOf(SetupApiException.class) + .extracting("errorCode").isEqualTo(SetupErrorCode.OPERATION_CONFLICT); + verify(fixture.inspector, never()).inspect(any(), any(), any()); + verify(fixture.commands, never()).migrate(any()); + } + + @Test + void externalApplyAndExportFailBeforeSecretStoreOrRendererWork() { + Fixture fixture = fixture(); + + assertThatThrownBy(() -> fixture.workflow.migrate(request(OPERATION, ApplyMode.EXTERNAL_APPLY))) + .isInstanceOf(SetupApiException.class) + .hasNoCause(); + assertThatThrownBy(() -> fixture.workflow.prepareExport(OPERATION, mock( + org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationExportRequest.class))) + .isInstanceOf(SetupApiException.class) + .hasNoCause(); + + verify(fixture.commands, never()).migration(any()); + verify(fixture.inspector, never()).inspect(any(), any(), any()); + } + + @Test + void validationOwnsOnlyItsSecretCopyAndMapsInspectionWithoutThrowing() { + Fixture fixture = fixture(); + AtomicReference borrowed = new AtomicReference<>(); + when(fixture.inspector.inspect(any(), any(), any())).thenAnswer(invocation -> { + borrowed.set(invocation.getArgument(1)); + return TargetInspection.NON_EMPTY; + }); + MetadataMigrationValidationRequest request = validationRequestFixture(); + + var result = fixture.workflow.validate(request); + + assertThat(result.valid()).isFalse(); + assertThat(result.errorCode()).isEqualTo(SetupErrorCode.MIGRATION_TARGET_NOT_EMPTY); + assertThat(borrowed.get().copy()).containsOnly('\0'); + } + + @Test + void errorIdentityAndInterruptArePreservedByTheWorkflowBoundary() { + Fixture fixture = fixture(); + AssertionError fatal = new AssertionError("fatal"); + when(fixture.commands.migration(OPERATION)).thenThrow(fatal); + Thread.currentThread().interrupt(); + try { + assertThatThrownBy(() -> fixture.workflow.migrate(request(OPERATION, ApplyMode.MANAGED_WRITE))) + .isSameAs(fatal); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } finally { + Thread.interrupted(); + } + } + + @Test + void eachInspectionSettlesExactPendingCleanupWithTheSameAbsoluteDeadline() { + Fixture fixture = fixture(); + when(fixture.inspector.inspect(any(), any(), any())) + .thenReturn(TargetInspection.EMPTY); + + assertThat(fixture.workflow.validate(validationRequestFixture()).valid()).isTrue(); + + InOrder order = inOrder(fixture.inspector); + order.verify(fixture.inspector).retryCleanup(any()); + order.verify(fixture.inspector).inspect(any(), any(), any()); + var cleanupDeadline = org.mockito.ArgumentCaptor.forClass( + JdbcMetadataMigrationDeadline.class); + var inspectDeadline = org.mockito.ArgumentCaptor.forClass( + JdbcMetadataMigrationDeadline.class); + verify(fixture.inspector).retryCleanup(cleanupDeadline.capture()); + verify(fixture.inspector).inspect(any(), any(), inspectDeadline.capture()); + assertThat(inspectDeadline.getValue()).isSameAs(cleanupDeadline.getValue()); + } + + @Test + void failedCleanupBlocksTheNextAcquireAndPreservesErrorAndInterrupt() { + Fixture fixture = fixture(); + AssertionError fatal = new AssertionError("fatal-cleanup"); + org.mockito.Mockito.doThrow(new TargetJdbcConnectionException( + TargetJdbcConnectionErrorCode.CLEANUP_REQUIRED)) + .doThrow(fatal) + .doNothing() + .when(fixture.inspector).retryCleanup(any()); + when(fixture.inspector.inspect(any(), any(), any())).thenReturn(TargetInspection.EMPTY); + + assertThatThrownBy(() -> fixture.workflow.validate(validationRequestFixture())) + .isInstanceOfSatisfying(SetupApiException.class, + failure -> assertThat(failure.errorCode()) + .isEqualTo(SetupErrorCode.MIGRATION_UNAVAILABLE)); + verify(fixture.inspector, never()).inspect(any(), any(), any()); + Thread.currentThread().interrupt(); + try { + assertThatThrownBy(() -> fixture.workflow.validate(validationRequestFixture())) + .isSameAs(fatal); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } finally { + Thread.interrupted(); + } + assertThat(fixture.workflow.validate(validationRequestFixture()).valid()).isTrue(); + verify(fixture.inspector).inspect(any(), any(), any()); + } + + @Test + void activationRechecksDynamicOwnerAdmissionWithoutReleasingTheRetainedOperation() { + Fixture fixture = fixture(); + DeploymentView invalidOwner = new DeploymentView( + NOW, deploymentFixture().managementDatabase(), deploymentFixture().greptimeDatabase(), + ApplyMode.MANAGED_WRITE, MaintenanceMode.INACTIVE, + DeploymentTopology.UNKNOWN, + MigrationCapability.blocked( + SetupErrorCode.MIGRATION_TOPOLOGY_UNAVAILABLE, + MaintenanceAdmission.NOT_APPLICABLE)); + MigrationView awaiting = MigrationTestSnapshots.awaitingRestart(OPERATION); + when(fixture.projector.project()).thenReturn(invalidOwner).thenReturn(deploymentFixture()); + when(fixture.commands.activate(any(), any())).thenReturn(awaiting); + + assertThatThrownBy(() -> fixture.workflow.activate(OPERATION, + new org.apache.hertzbeat.manager.setup.api.DeploymentApiContract + .ActivateMigrationRequest( + org.apache.hertzbeat.manager.setup.api.DeploymentApiContract + .MigrationOperationState.READY_TO_ACTIVATE))) + .isInstanceOfSatisfying(SetupApiException.class, + failure -> assertThat(failure.errorCode()) + .isEqualTo(SetupErrorCode.MIGRATION_UNAVAILABLE)); + verify(fixture.commands, never()).activate(any(), any()); + assertThat(fixture.workflow.activate(OPERATION, + new org.apache.hertzbeat.manager.setup.api.DeploymentApiContract + .ActivateMigrationRequest( + org.apache.hertzbeat.manager.setup.api.DeploymentApiContract + .MigrationOperationState.READY_TO_ACTIVATE))).isSameAs(awaiting); + verify(fixture.commands).activate(any(), any()); + } + + @Test + void blockedPendingStillExecutingReplaysWithoutAnotherTargetInspection() { + Fixture fixture = fixture(); + MigrationView blocked = MigrationTestSnapshots.blockedPending(OPERATION); + when(fixture.commands.migration(OPERATION)).thenReturn(blocked); + when(fixture.commands.joinExecuting(any())).thenReturn(Optional.of(blocked)); + + assertThat(fixture.workflow.migrate(request(OPERATION, ApplyMode.MANAGED_WRITE))) + .isSameAs(blocked); + + verify(fixture.inspector, never()).inspect(any(), any(), any()); + verify(fixture.commands).joinExecuting(any()); + } + + @Test + void pendingThatFinishesBeforeJoinIsRereadWithoutTargetInspection() { + Fixture fixture = fixture(); + MigrationView pending = MigrationTestSnapshots.blockedPending(OPERATION); + MigrationView ready = mock(MigrationView.class); + when(ready.state()).thenReturn(MigrationOperationState.READY_TO_ACTIVATE); + when(ready.target()).thenReturn(MigrationTarget.POSTGRESQL); + when(fixture.commands.migration(OPERATION)).thenReturn(pending, ready); + when(fixture.commands.joinExecuting(any())).thenReturn(Optional.empty()); + when(fixture.inspector.inspect(any(), any(), any())) + .thenThrow(new AssertionError("target inspection must not restart")); + + assertThat(fixture.workflow.migrate(request(OPERATION, ApplyMode.MANAGED_WRITE))) + .isSameAs(ready); + + verify(fixture.commands, times(2)).migration(OPERATION); + verify(fixture.inspector, never()).inspect(any(), any(), any()); + verify(fixture.commands, never()).migrate(any()); + } + + @Test + @Timeout(10) + void closeSealsAdmissionWhileSerializedMigrateIsStillBlocked() throws Exception { + Fixture fixture = fixture(); + MigrationView running = MigrationTestSnapshots.running(OPERATION); + CountDownLatch inspectionEntered = new CountDownLatch(1); + CountDownLatch releaseInspection = new CountDownLatch(1); + AtomicInteger inspections = new AtomicInteger(); + AtomicReference closeFailure = new AtomicReference<>(); + when(fixture.commands.migration(OPERATION)) + .thenThrow(new MigrationOperationStoreException(SetupErrorCode.OPERATION_NOT_FOUND)); + when(fixture.commands.activeOperationId()).thenReturn(Optional.empty()); + when(fixture.inspector.inspect(any(), any(), any())).thenAnswer(invocation -> { + inspections.incrementAndGet(); + inspectionEntered.countDown(); + assertThat(releaseInspection.await(5, TimeUnit.SECONDS)).isTrue(); + return TargetInspection.EMPTY; + }); + when(fixture.commands.migrate(any())).thenReturn(running); + Thread closer = Thread.ofPlatform().unstarted(() -> { + try { + fixture.workflow.closeAdmission(); + } catch (Throwable failure) { + closeFailure.set(failure); + } + }); + + try (var callers = Executors.newVirtualThreadPerTaskExecutor()) { + Future first = callers.submit( + () -> fixture.workflow.migrate(request(OPERATION, ApplyMode.MANAGED_WRITE))); + assertThat(inspectionEntered.await(5, TimeUnit.SECONDS)).isTrue(); + closer.start(); + awaitCondition(() -> closer.getState() == Thread.State.BLOCKED + || closer.getState() == Thread.State.WAITING + || closer.getState() == Thread.State.TIMED_WAITING); + + Future validation = callers.submit(() -> captureFailure( + () -> fixture.workflow.validate(validationRequestFixture()))); + Future activation = callers.submit(() -> captureFailure( + () -> fixture.workflow.activate(OPERATION, new ActivateMigrationRequest( + org.apache.hertzbeat.manager.setup.api.DeploymentApiContract + .MigrationOperationState.READY_TO_ACTIVATE)))); + + assertUnavailable(validation.get(1, TimeUnit.SECONDS)); + assertUnavailable(activation.get(1, TimeUnit.SECONDS)); + assertThat(inspections.get()).isOne(); + assertThat(closer.isAlive()).isTrue(); + + releaseInspection.countDown(); + assertThat(first.get(5, TimeUnit.SECONDS)).isSameAs(running); + closer.join(TimeUnit.SECONDS.toMillis(5)); + assertThat(closer.isAlive()).isFalse(); + assertThat(closeFailure.get()).isNull(); + verify(fixture.inspector).inspect(any(), any(), any()); + verify(fixture.commands, never()).activate(any(), any()); + } finally { + releaseInspection.countDown(); + if (closer.isAlive()) { + closer.interrupt(); + closer.join(TimeUnit.SECONDS.toMillis(5)); + } + } + } + + private static Throwable captureFailure(Runnable action) { + try { + action.run(); + return null; + } catch (Throwable failure) { + return failure; + } + } + + private static void assertUnavailable(Throwable failure) { + assertThat(failure).isInstanceOfSatisfying(SetupApiException.class, + unavailable -> assertThat(unavailable.errorCode()) + .isEqualTo(SetupErrorCode.MIGRATION_UNAVAILABLE)); + } + + private static void awaitCondition(BooleanSupplier condition) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (!condition.getAsBoolean() && System.nanoTime() - deadline < 0) { + Thread.sleep(1); + } + assertThat(condition.getAsBoolean()).isTrue(); + } + + private Fixture fixture() { + ManagedDeploymentMigrationCommands commands = mock(ManagedDeploymentMigrationCommands.class); + MetadataMigrationTargetInspector inspector = mock(MetadataMigrationTargetInspector.class); + DeploymentViewProjector projector = mock(DeploymentViewProjector.class); + when(projector.project()).thenReturn(deploymentFixture()); + DefaultDeploymentWorkflow workflow = new DefaultDeploymentWorkflow( + projector, commands, inspector, new MetadataMigrationPolicy(), + new DeploymentWorkflowFailureMapper(), Clock.fixed(NOW, ZoneOffset.UTC), + TIMEOUT, System::nanoTime); + return new Fixture(commands, inspector, projector, workflow); + } + + private static MetadataMigrationRequest request(String operationId, ApplyMode mode) { + return new MetadataMigrationRequest( + operationId, MigrationTarget.POSTGRESQL, database(), mode); + } + + private static MetadataDatabaseConfiguration database() { + return new MetadataDatabaseConfiguration( + MetadataDatabaseKind.POSTGRESQL, "jdbc:postgresql://db.example:5432/hertzbeat", + "operator", "private-password"); + } + + static MetadataMigrationValidationRequest validationRequestFixture() { + return new MetadataMigrationValidationRequest(MigrationTarget.POSTGRESQL, database()); + } + + static DeploymentView deploymentFixture() { + return new DeploymentView( + NOW, + new ManagementDatabaseSummary(MetadataDatabaseKind.H2, true, ConfigSource.UI_MANAGED, false), + new TelemetryStoreSummary(TelemetryStoreKind.GREPTIME, true, ConfigSource.UI_MANAGED, false), + ApplyMode.MANAGED_WRITE, + MaintenanceMode.INACTIVE, + DeploymentTopology.SINGLE_NODE, + MigrationCapability.permitted(MaintenanceAdmission.AUTO_ENTER)); + } + + private record Fixture( + ManagedDeploymentMigrationCommands commands, + MetadataMigrationTargetInspector inspector, + DeploymentViewProjector projector, + DefaultDeploymentWorkflow workflow) { + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationConfigurationTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationConfigurationTest.java new file mode 100644 index 0000000000..7a93129d30 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationConfigurationTest.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.nio.file.Path; +import java.time.Clock; +import java.util.List; +import org.apache.hertzbeat.common.runtime.BusinessRuntimeGate; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceCoordinator; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceOrchestrator; +import org.apache.hertzbeat.manager.maintenance.StandaloneDeploymentOwnerView; +import org.apache.hertzbeat.manager.setup.api.DeploymentWorkflow; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupAccess; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.config.DeploymentConstraint; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; +import org.apache.hertzbeat.manager.setup.config.SetupInstallationPaths; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; + +class DeploymentMigrationConfigurationTest { + + @TempDir + private Path root; + + @Test + void validNormalManagedOwnerOpensOneRuntimeAndExposesOneWorkflow() { + DeploymentMigrationRuntime.Opener opener = mock(DeploymentMigrationRuntime.Opener.class); + DeploymentMigrationRuntime runtime = mock(DeploymentMigrationRuntime.class); + DefaultDeploymentWorkflow workflow = mock(DefaultDeploymentWorkflow.class); + when(runtime.workflow()).thenReturn(workflow); + when(runtime.available()).thenReturn(true); + when(opener.open(any())).thenReturn(runtime); + + context(RuntimeMode.NORMAL, writable(), owner(root), opener).run(result -> { + assertThat(result).hasNotFailed(); + assertThat(result).hasSingleBean(DeploymentMigrationRuntime.class); + assertThat(result).hasSingleBean(DeploymentWorkflow.class); + assertThat(result.getBean(DeploymentWorkflow.class)).isSameAs(workflow); + verify(opener).open(any()); + }); + } + + @Test + void configurationIsAbsentOutsideNormalWithoutOpeningRuntime() { + for (RuntimeMode mode : List.of( + RuntimeMode.SETUP_ONLY, RuntimeMode.FULL_SETUP_GATED, RuntimeMode.RECOVERY)) { + DeploymentMigrationRuntime.Opener opener = mock(DeploymentMigrationRuntime.Opener.class); + context(mode, writable(), owner(root), opener).run(result -> { + assertThat(result).doesNotHaveBean(DeploymentMigrationRuntime.class); + assertThat(result).doesNotHaveBean(DeploymentWorkflow.class); + verify(opener, never()).open(any()); + }); + } + } + + @Test + void closedGateInvalidOwnerAndReadOnlyCapabilityPerformNoRuntimeIo() { + DeploymentMigrationRuntime.Opener closedGate = mock(DeploymentMigrationRuntime.Opener.class); + context(RuntimeMode.NORMAL, RuntimeMode.FULL_SETUP_GATED, writable(), owner(root), closedGate) + .run(result -> assertUnavailable(result, closedGate)); + + StandaloneDeploymentOwnerView invalid = owner(root); + when(invalid.isValid()).thenReturn(false); + DeploymentMigrationRuntime.Opener invalidOwner = mock(DeploymentMigrationRuntime.Opener.class); + context(RuntimeMode.NORMAL, writable(), invalid, invalidOwner) + .run(result -> assertUnavailable(result, invalidOwner)); + + DeploymentMigrationRuntime.Opener readOnly = mock(DeploymentMigrationRuntime.Opener.class); + context(RuntimeMode.NORMAL, new ManagedConfigCapability( + ApplyMode.EXTERNAL_APPLY, false, DeploymentConstraint.READ_ONLY), owner(root), readOnly) + .run(result -> assertUnavailable(result, readOnly)); + } + + @Test + void mismatchedOwnerRootPerformsNoRuntimeIo() { + DeploymentMigrationRuntime.Opener opener = mock(DeploymentMigrationRuntime.Opener.class); + StandaloneDeploymentOwnerView owner = owner(root.resolve("foreign")); + + context(RuntimeMode.NORMAL, writable(), owner, opener) + .run(result -> assertUnavailable(result, opener)); + } + + private static void assertUnavailable( + org.springframework.boot.test.context.assertj.AssertableApplicationContext context, + DeploymentMigrationRuntime.Opener opener) { + assertThat(context).doesNotHaveBean(DeploymentWorkflow.class); + DeploymentMigrationRuntime runtime = context.getBean(DeploymentMigrationRuntime.class); + assertThat(runtime.workflow().deployment().migration().allowed()).isFalse(); + assertThat(runtime.workflow().deployment().migration().blockedBy()) + .isEqualTo(org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode + .MIGRATION_UNAVAILABLE); + verify(opener, never()).open(any()); + } + + private ApplicationContextRunner context( + RuntimeMode propertyMode, + ManagedConfigCapability capability, + StandaloneDeploymentOwnerView owner, + DeploymentMigrationRuntime.Opener opener) { + return context(propertyMode, propertyMode, capability, owner, opener); + } + + private ApplicationContextRunner context( + RuntimeMode propertyMode, + RuntimeMode gateMode, + ManagedConfigCapability capability, + StandaloneDeploymentOwnerView owner, + DeploymentMigrationRuntime.Opener opener) { + return new ApplicationContextRunner() + .withPropertyValues( + RuntimeMode.PROPERTY_NAME + "=" + propertyMode.value(), + SetupInstallationPaths.ROOT_PROPERTY + "=" + root) + .withUserConfiguration(DeploymentMigrationConfiguration.class) + .withBean(BusinessRuntimeGate.class, () -> BusinessRuntimeGate.fixed(gateMode)) + .withBean(ManagedConfigCapability.class, () -> capability) + .withBean(StandaloneDeploymentOwnerView.class, () -> owner) + .withBean(SetupRuntimeState.class, () -> new SetupRuntimeState( + Clock.systemUTC(), capability, SetupPhase.COMPLETE, + SetupAccess.LOCAL, true, "admin")) + .withBean(MetadataMaintenanceCoordinator.class, + () -> mock(MetadataMaintenanceCoordinator.class)) + .withBean(MigrationMaintenanceOrchestrator.class, + () -> mock(MigrationMaintenanceOrchestrator.class)) + .withBean(DeploymentMigrationRuntime.Opener.class, () -> opener); + } + + private static StandaloneDeploymentOwnerView owner(Path root) { + StandaloneDeploymentOwnerView owner = mock(StandaloneDeploymentOwnerView.class); + when(owner.installationRoot()).thenReturn(root.toAbsolutePath().normalize()); + when(owner.isValid()).thenReturn(true); + return owner; + } + + private static ManagedConfigCapability writable() { + return new ManagedConfigCapability(ApplyMode.MANAGED_WRITE, true, DeploymentConstraint.NONE); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationRuntimeTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationRuntimeTest.java new file mode 100644 index 0000000000..c4db1d0fc4 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentMigrationRuntimeTest.java @@ -0,0 +1,267 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.config.MetadataDatabaseSettings; +import org.apache.hertzbeat.manager.setup.config.SecretValue; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.mockito.InOrder; + +class DeploymentMigrationRuntimeTest { + + @Test + void closeRetainsTheExactPhaseForAnExplicitRetry() { + DefaultDeploymentWorkflow workflow = mock(DefaultDeploymentWorkflow.class); + ManagedDeploymentMigrationCommands commands = mock(ManagedDeploymentMigrationCommands.class); + MetadataMigrationTargetInspector inspector = mock(MetadataMigrationTargetInspector.class); + JdbcMetadataMigrationExecutor copyExecutor = mock(JdbcMetadataMigrationExecutor.class); + TargetJdbcConnectionFactory factory = mock(TargetJdbcConnectionFactory.class); + RuntimeException first = new IllegalStateException("private"); + doThrow(first).doNothing().when(commands).close(); + DeploymentMigrationRuntime runtime = new DeploymentMigrationRuntime( + workflow, commands, inspector, copyExecutor, factory, + Duration.ofSeconds(5), System::nanoTime); + + assertThatThrownBy(runtime::close).isSameAs(first); + verify(inspector, never()).shutdown(any()); + runtime.close(); + + InOrder order = inOrder(workflow, commands, inspector, copyExecutor, factory); + order.verify(workflow).closeAdmission(); + order.verify(commands, times(2)).close(); + order.verify(inspector).shutdown(any()); + order.verify(copyExecutor).close(); + order.verify(factory).close(); + assertThat(runtime.workflow()).isSameAs(workflow); + } + + @Test + void springDestroyRetriesTheExactPhaseUntilItSettlesWithinOneCallback() { + DefaultDeploymentWorkflow workflow = mock(DefaultDeploymentWorkflow.class); + ManagedDeploymentMigrationCommands commands = mock(ManagedDeploymentMigrationCommands.class); + RuntimeException first = new IllegalStateException("private"); + doThrow(first).doThrow(first).doNothing().when(commands).close(); + MetadataMigrationTargetInspector inspector = mock(MetadataMigrationTargetInspector.class); + DeploymentMigrationRuntime runtime = new DeploymentMigrationRuntime( + workflow, commands, inspector, mock(JdbcMetadataMigrationExecutor.class), + mock(TargetJdbcConnectionFactory.class), Duration.ofSeconds(5), System::nanoTime); + + runtime.destroySafely(); + + verify(workflow).closeAdmission(); + verify(commands, times(3)).close(); + verify(inspector).shutdown(any()); + } + + @Test + @Timeout(10) + void springDestroyBacksOffAfterItsRetryBudgetAndKeepsOwningTheExactPhase() + throws Exception { + DefaultDeploymentWorkflow workflow = mock(DefaultDeploymentWorkflow.class); + ManagedDeploymentMigrationCommands commands = mock(ManagedDeploymentMigrationCommands.class); + MetadataMigrationTargetInspector inspector = mock(MetadataMigrationTargetInspector.class); + CountDownLatch firstAttempt = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch hotLoop = new CountDownLatch(100); + AtomicInteger attempts = new AtomicInteger(); + doAnswer(ignored -> { + attempts.incrementAndGet(); + firstAttempt.countDown(); + hotLoop.countDown(); + if (release.getCount() != 0) { + throw new IllegalStateException("private"); + } + return null; + }).when(commands).close(); + DeploymentMigrationRuntime runtime = new DeploymentMigrationRuntime( + workflow, commands, inspector, mock(JdbcMetadataMigrationExecutor.class), + mock(TargetJdbcConnectionFactory.class), Duration.ofMillis(10), System::nanoTime); + + try (var caller = Executors.newVirtualThreadPerTaskExecutor()) { + Future destroying = caller.submit(runtime::destroySafely); + assertThat(firstAttempt.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(hotLoop.await(200, TimeUnit.MILLISECONDS)).isFalse(); + assertThat(attempts.get()).isLessThan(100); + verify(inspector, never()).shutdown(any()); + release.countDown(); + destroying.get(5, TimeUnit.SECONDS); + } finally { + release.countDown(); + } + verify(inspector).shutdown(any()); + } + + @Test + void errorIdentityAndInterruptSurviveRetryableClose() { + ManagedDeploymentMigrationCommands commands = mock(ManagedDeploymentMigrationCommands.class); + MetadataMigrationTargetInspector inspector = mock(MetadataMigrationTargetInspector.class); + AssertionError fatal = new AssertionError("fatal"); + doThrow(fatal).doNothing().when(inspector).shutdown(any()); + DeploymentMigrationRuntime runtime = new DeploymentMigrationRuntime( + mock(DefaultDeploymentWorkflow.class), commands, inspector, + mock(JdbcMetadataMigrationExecutor.class), mock(TargetJdbcConnectionFactory.class), + Duration.ofSeconds(5), System::nanoTime); + Thread.currentThread().interrupt(); + try { + assertThatThrownBy(runtime::close).isSameAs(fatal); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + runtime.close(); + } finally { + Thread.interrupted(); + } + } + + @Test + void constructionCleanupErrorOutranksAnOrdinaryFailureAndPreservesInterrupt() { + RuntimeException construction = new IllegalStateException("private-construction"); + AssertionError cleanupFatal = new AssertionError("private-cleanup"); + AtomicBoolean cleanupRan = new AtomicBoolean(); + Thread.currentThread().interrupt(); + try { + Throwable first = DeploymentMigrationRuntime.runCleanupAfter(construction, () -> { + assertThat(Thread.currentThread().isInterrupted()).isFalse(); + cleanupRan.set(true); + Thread.currentThread().interrupt(); + throw cleanupFatal; + }); + + assertThat(first).isSameAs(cleanupFatal); + assertThat(first.getSuppressed()).hasSize(1); + assertThat(first.getSuppressed()[0]) + .isInstanceOf(MigrationOperationStoreException.class) + .hasNoCause(); + assertThat(cleanupRan).isTrue(); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } finally { + Thread.interrupted(); + } + } + + @Test + @Timeout(10) + void closeSealsWorkflowBeforeWaitingForAnInFlightValidationAndClosingCommands() + throws Exception { + ManagedDeploymentMigrationCommands commands = mock(ManagedDeploymentMigrationCommands.class); + MetadataMigrationTargetInspector inspector = mock(MetadataMigrationTargetInspector.class); + DeploymentViewProjector projector = mock(DeploymentViewProjector.class); + when(projector.project()).thenReturn(DefaultDeploymentWorkflowTest.deploymentFixture()); + CountDownLatch inspectionEntered = new CountDownLatch(1); + CountDownLatch releaseInspection = new CountDownLatch(1); + when(inspector.inspect(any(), any(), any())).thenAnswer(ignored -> { + inspectionEntered.countDown(); + assertThat(releaseInspection.await(5, TimeUnit.SECONDS)).isTrue(); + return org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.TargetInspection.EMPTY; + }); + DefaultDeploymentWorkflow workflow = new DefaultDeploymentWorkflow( + projector, commands, inspector, new MetadataMigrationPolicy(), + new DeploymentWorkflowFailureMapper(), java.time.Clock.systemUTC(), + Duration.ofSeconds(5), System::nanoTime); + DeploymentMigrationRuntime runtime = new DeploymentMigrationRuntime( + workflow, commands, inspector, mock(JdbcMetadataMigrationExecutor.class), + mock(TargetJdbcConnectionFactory.class), Duration.ofSeconds(5), System::nanoTime); + + try (var callers = Executors.newVirtualThreadPerTaskExecutor()) { + Future validation = callers.submit(() -> workflow.validate( + DefaultDeploymentWorkflowTest.validationRequestFixture())); + assertThat(inspectionEntered.await(5, TimeUnit.SECONDS)).isTrue(); + Future closing = callers.submit(runtime::close); + assertThatThrownBy(() -> workflow.migration("operation-a")) + .isInstanceOf(org.apache.hertzbeat.manager.setup.api.SetupApiException.class); + verify(commands, never()).close(); + releaseInspection.countDown(); + validation.get(5, TimeUnit.SECONDS); + closing.get(5, TimeUnit.SECONDS); + } finally { + releaseInspection.countDown(); + } + verify(commands).close(); + } + + @Test + @Timeout(10) + void lateConnectionAfterRuntimeAndFactoryCloseStillUsesProcessLifetimeAbortAndExactClose() + throws Exception { + CountDownLatch connectorEntered = new CountDownLatch(1); + CountDownLatch releaseConnector = new CountDownLatch(1); + CountDownLatch aborted = new CountDownLatch(1); + CountDownLatch closed = new CountDownLatch(1); + Connection connection = mock(Connection.class); + org.mockito.Mockito.doAnswer(ignored -> { + aborted.countDown(); + return null; + }).when(connection).abort(any()); + org.mockito.Mockito.doAnswer(ignored -> { + closed.countDown(); + return null; + }).when(connection).close(); + TargetJdbcConnector connector = (target, username, password, deadline) -> { + connectorEntered.countDown(); + boolean interrupted = false; + while (releaseConnector.getCount() != 0) { + try { + releaseConnector.await(); + } catch (InterruptedException ignored) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + return connection; + }; + TargetJdbcConnectionFactory factory = new TargetJdbcConnectionFactory( + connector, TargetJdbcAbortExecutor.instance()); + DeploymentMigrationRuntime runtime = new DeploymentMigrationRuntime( + mock(DefaultDeploymentWorkflow.class), mock(ManagedDeploymentMigrationCommands.class), + mock(MetadataMigrationTargetInspector.class), mock(JdbcMetadataMigrationExecutor.class), + factory, Duration.ofMillis(20), System::nanoTime); + + try (var caller = Executors.newVirtualThreadPerTaskExecutor(); + SecretValue password = SecretValue.of("private-password")) { + Future acquisition = caller.submit(() -> factory.acquire( + new MetadataDatabaseSettings( + MetadataDatabaseKind.POSTGRESQL, + "jdbc:postgresql://db.example:5432/hertzbeat", "operator"), + password, + JdbcMetadataMigrationDeadline.start(Duration.ofMillis(20), System::nanoTime))); + assertThat(connectorEntered.await(5, TimeUnit.SECONDS)).isTrue(); + assertThatThrownBy(() -> acquisition.get(5, TimeUnit.SECONDS)).isNotNull(); + + runtime.close(); + releaseConnector.countDown(); + assertThat(aborted.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(closed.await(5, TimeUnit.SECONDS)).isTrue(); + } finally { + releaseConnector.countDown(); + } + verify(connection).abort(any()); + verify(connection).close(); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentViewProjectorTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentViewProjectorTest.java new file mode 100644 index 0000000000..434a9ab95b --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentViewProjectorTest.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.nio.file.Path; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.List; +import java.util.Optional; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceCoordinator; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenancePhase; +import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceSnapshot; +import org.apache.hertzbeat.manager.maintenance.StandaloneDeploymentOwnerView; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.DeploymentTopology; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MaintenanceAdmission; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MaintenanceMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ConfigSource; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ManagementDatabaseSummary; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.OptionalConfigurationSummary; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupAccess; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupPhase; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreKind; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.TelemetryStoreSummary; +import org.apache.hertzbeat.manager.setup.config.DeploymentConstraint; +import org.apache.hertzbeat.manager.setup.config.ManagedConfigCapability; +import org.junit.jupiter.api.Test; + +class DeploymentViewProjectorTest { + + private static final String ACTIVE = "operation-a"; + private static final Path ROOT = Path.of(".").toAbsolutePath().normalize(); + + @Test + void validManagedH2OwnerIsSingleNodeAndCanAutoEnterMaintenance() { + Fixture fixture = fixture(MetadataDatabaseKind.H2, writable()); + + var view = fixture.projector.project(); + + assertThat(view.topology()).isEqualTo(DeploymentTopology.SINGLE_NODE); + assertThat(view.maintenanceMode()).isEqualTo(MaintenanceMode.INACTIVE); + assertThat(view.migration().allowed()).isTrue(); + assertThat(view.migration().maintenanceAdmission()).isEqualTo(MaintenanceAdmission.AUTO_ENTER); + } + + @Test + void durableActiveOperationAndMaintenanceOwnershipAreProjectedAsConflict() { + Fixture fixture = fixture(MetadataDatabaseKind.H2, writable()); + when(fixture.commands.activeOperationId()).thenReturn(Optional.of(ACTIVE)); + when(fixture.maintenance.snapshot()).thenReturn( + new MetadataMaintenanceSnapshot(MetadataMaintenancePhase.QUIESCED, ACTIVE, 2)); + + var capability = fixture.projector.project().migration(); + + assertThat(capability.allowed()).isFalse(); + assertThat(capability.blockedBy()).isEqualTo(SetupErrorCode.OPERATION_CONFLICT); + assertThat(capability.activeOperationId()).isEqualTo(ACTIVE); + } + + @Test + void invalidOwnerAndNonH2SourceUseStructuralFailClosedBlockers() { + Fixture invalidOwner = fixture(MetadataDatabaseKind.H2, writable()); + when(invalidOwner.owner.isValid()).thenReturn(false); + assertThat(invalidOwner.projector.project().migration().blockedBy()) + .isEqualTo(SetupErrorCode.MIGRATION_TOPOLOGY_UNAVAILABLE); + + Fixture nonH2 = fixture(MetadataDatabaseKind.POSTGRESQL, writable()); + assertThat(nonH2.projector.project().migration().blockedBy()) + .isEqualTo(SetupErrorCode.MIGRATION_SOURCE_UNSUPPORTED); + } + + @Test + void externalConfigurationAndUnsettledMaintenanceRemainUnavailable() { + Fixture external = fixture(MetadataDatabaseKind.H2, new ManagedConfigCapability( + ApplyMode.EXTERNAL_APPLY, false, DeploymentConstraint.READ_ONLY)); + assertThat(external.projector.project().migration().blockedBy()) + .isEqualTo(SetupErrorCode.MIGRATION_UNAVAILABLE); + + Fixture recovering = fixture(MetadataDatabaseKind.H2, writable()); + when(recovering.maintenance.snapshot()).thenReturn( + new MetadataMaintenanceSnapshot(MetadataMaintenancePhase.RECOVERY_REQUIRED, ACTIVE, 3)); + assertThat(recovering.projector.project().migration().blockedBy()) + .isEqualTo(SetupErrorCode.MIGRATION_UNAVAILABLE); + } + + private static Fixture fixture( + MetadataDatabaseKind kind, ManagedConfigCapability capability) { + SetupConfigurationProjection configuration = new SetupConfigurationProjection( + new ManagementDatabaseSummary(kind, true, ConfigSource.UI_MANAGED, false), + new TelemetryStoreSummary( + TelemetryStoreKind.GREPTIME, true, ConfigSource.UI_MANAGED, false), + new OptionalConfigurationSummary(false, false, false, false, false), List.of()); + SetupRuntimeState state = new SetupRuntimeState( + Clock.fixed(Instant.parse("2026-08-10T01:00:00Z"), ZoneOffset.UTC), + capability, SetupPhase.COMPLETE, SetupAccess.LOCAL, true, "admin", configuration); + StandaloneDeploymentOwnerView owner = mock(StandaloneDeploymentOwnerView.class); + when(owner.installationRoot()).thenReturn(ROOT); + when(owner.isValid()).thenReturn(true); + MetadataMaintenanceCoordinator maintenance = mock(MetadataMaintenanceCoordinator.class); + when(maintenance.snapshot()).thenReturn( + new MetadataMaintenanceSnapshot(MetadataMaintenancePhase.RUNNING, null, 1)); + ManagedDeploymentMigrationCommands commands = mock(ManagedDeploymentMigrationCommands.class); + when(commands.activeOperationId()).thenReturn(Optional.empty()); + DeploymentViewProjector projector = new DeploymentViewProjector( + state, capability, owner, maintenance, commands, + Clock.fixed(Instant.parse("2026-08-10T02:00:00Z"), ZoneOffset.UTC)); + return new Fixture(owner, maintenance, commands, projector); + } + + private static ManagedConfigCapability writable() { + return new ManagedConfigCapability(ApplyMode.MANAGED_WRITE, true, DeploymentConstraint.NONE); + } + + private record Fixture( + StandaloneDeploymentOwnerView owner, + MetadataMaintenanceCoordinator maintenance, + ManagedDeploymentMigrationCommands commands, + DeploymentViewProjector projector) { + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentWorkflowFailureMapperTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentWorkflowFailureMapperTest.java new file mode 100644 index 0000000000..6400f0845c --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/DeploymentWorkflowFailureMapperTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode; +import org.apache.hertzbeat.manager.setup.api.SetupApiException; +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpStatus; + +class DeploymentWorkflowFailureMapperTest { + + private final DeploymentWorkflowFailureMapper mapper = new DeploymentWorkflowFailureMapper(); + + @Test + void stableStoreFailuresBecomeCauseFreeHttpClassifications() { + assertMapped(SetupErrorCode.INVALID_REQUEST, HttpStatus.BAD_REQUEST); + assertMapped(SetupErrorCode.OPERATION_NOT_FOUND, HttpStatus.NOT_FOUND); + assertMapped(SetupErrorCode.OPERATION_CONFLICT, HttpStatus.CONFLICT); + assertMapped(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, HttpStatus.SERVICE_UNAVAILABLE); + assertMapped(SetupErrorCode.MIGRATION_UNAVAILABLE, HttpStatus.SERVICE_UNAVAILABLE); + } + + @Test + void everyOrdinaryRuntimeIsCauseFreeAndMetadataCodesRemainStable() { + RuntimeException unknown = new IllegalStateException("private detail"); + SetupApiException api = new SetupApiException(SetupErrorCode.INVALID_REQUEST, HttpStatus.BAD_REQUEST); + + assertTranslated(unknown, SetupErrorCode.MIGRATION_UNAVAILABLE, HttpStatus.SERVICE_UNAVAILABLE); + assertThat(mapper.translate(api)).isSameAs(api); + assertTranslated(new MetadataMigrationException(MetadataMigrationErrorCode.SCHEMA), + SetupErrorCode.MIGRATION_COPY_FAILED, HttpStatus.CONFLICT); + assertTranslated(new MetadataMigrationException(MetadataMigrationErrorCode.COPY), + SetupErrorCode.MIGRATION_COPY_FAILED, HttpStatus.CONFLICT); + assertTranslated(new MetadataMigrationException(MetadataMigrationErrorCode.VERIFICATION), + SetupErrorCode.MIGRATION_VERIFICATION_FAILED, HttpStatus.CONFLICT); + assertTranslated(new MetadataMigrationException(MetadataMigrationErrorCode.SEQUENCE), + SetupErrorCode.MIGRATION_VERIFICATION_FAILED, HttpStatus.CONFLICT); + assertTranslated(new MetadataMigrationException(MetadataMigrationErrorCode.TIMEOUT), + SetupErrorCode.MIGRATION_UNAVAILABLE, HttpStatus.SERVICE_UNAVAILABLE); + assertTranslated(new MetadataMigrationException( + MetadataMigrationErrorCode.COMMIT_OUTCOME_UNKNOWN), + SetupErrorCode.CONFIG_RECOVERY_REQUIRED, HttpStatus.SERVICE_UNAVAILABLE); + assertTranslated(new MetadataMigrationException( + MetadataMigrationErrorCode.ROLLBACK_OUTCOME_UNKNOWN), + SetupErrorCode.CONFIG_RECOVERY_REQUIRED, HttpStatus.SERVICE_UNAVAILABLE); + } + + private void assertMapped(SetupErrorCode code, HttpStatus status) { + assertTranslated(new MigrationOperationStoreException(code), code, status); + } + + private void assertTranslated( + RuntimeException failure, SetupErrorCode code, HttpStatus status) { + RuntimeException translated = mapper.translate(failure); + + assertThat(translated).isInstanceOf(SetupApiException.class).hasNoCause(); + SetupApiException api = (SetupApiException) translated; + assertThat(api.errorCode()).isEqualTo(code); + assertThat(api.status()).isEqualTo(status); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedDeploymentMigrationCommandsTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedDeploymentMigrationCommandsTest.java index 29fcacdb66..e319437252 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedDeploymentMigrationCommandsTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedDeploymentMigrationCommandsTest.java @@ -17,6 +17,7 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; import java.util.Optional; @@ -74,6 +75,115 @@ class ManagedDeploymentMigrationCommandsTest { order.verify(fixture.coordinator).status(); } + @Test + void executingAndHandoffInProgressReplayTheExactRunningJournal() { + Fixture fixture = fixture(); + MigrationView running = MigrationOperationProjection.view(running()); + fixture.store.create(pending()); + fixture.store.compareAndTransition( + OPERATION, MigrationOperationState.PENDING, running()); + when(fixture.runner.inFlightSnapshot(OPERATION)).thenReturn(Optional.of(running())); + when(fixture.coordinator.status()) + .thenReturn(new RetainedCutoverStatus( + OPERATION, RetainedCutoverStatus.Phase.EXECUTING)) + .thenReturn(new RetainedCutoverStatus( + OPERATION, RetainedCutoverStatus.Phase.HANDOFFING)); + + assertThat(fixture.commands.migration(OPERATION)).isEqualTo(running); + assertThat(fixture.commands.migration(OPERATION)).isEqualTo(running); + } + + @Test + void livePhaseWhoseTaskJustClearedReplaysTheConfirmedReadyJournal() { + Fixture fixture = fixture(); + advanceToReady(fixture.store); + when(fixture.runner.inFlightSnapshot(OPERATION)).thenReturn(Optional.empty()); + when(fixture.runner.find(OPERATION)).thenThrow( + new MigrationOperationStoreException(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)); + when(fixture.coordinator.status()) + .thenReturn(new RetainedCutoverStatus( + OPERATION, RetainedCutoverStatus.Phase.HANDOFFING)) + .thenReturn(new RetainedCutoverStatus( + OPERATION, RetainedCutoverStatus.Phase.RETAINED)); + + assertThat(fixture.commands.migration(OPERATION).state()) + .isEqualTo(MigrationOperationState.READY_TO_ACTIVATE); + verify(fixture.runner, never()).find(OPERATION); + } + + @Test + void handoffPhaseNeverExposesReadyBeforeTheCoordinatorConfirmsRetention() { + Fixture fixture = fixture(); + advanceToReady(fixture.store); + when(fixture.runner.inFlightSnapshot(OPERATION)).thenReturn(Optional.of(running())); + when(fixture.coordinator.status()).thenReturn(new RetainedCutoverStatus( + OPERATION, RetainedCutoverStatus.Phase.HANDOFFING)); + + MigrationView view = fixture.commands.migration(OPERATION); + + assertThat(view.state()).isEqualTo(MigrationOperationState.RUNNING); + assertThat(view.progressPercent()).isZero(); + } + + @Test + void retainedPhaseReplaysConfirmedReadyWhileTheWorkerIsStillUnwinding() { + Fixture fixture = fixture(); + advanceToReady(fixture.store); + when(fixture.runner.find(OPERATION)).thenThrow( + new MigrationOperationStoreException(SetupErrorCode.CONFIG_RECOVERY_REQUIRED)); + when(fixture.coordinator.status()).thenReturn(new RetainedCutoverStatus( + OPERATION, RetainedCutoverStatus.Phase.RETAINED)); + + assertThat(fixture.commands.migration(OPERATION).state()) + .isEqualTo(MigrationOperationState.READY_TO_ACTIVATE); + verify(fixture.runner, never()).find(OPERATION); + } + + @Test + void liveProjectionReadsCurrentProgressInsteadOfFreezingThePreparationView() { + Fixture fixture = fixture(); + fixture.store.create(pending()); + fixture.store.compareAndTransition( + OPERATION, MigrationOperationState.PENDING, running(0)); + fixture.store.compareAndTransition( + OPERATION, MigrationOperationState.RUNNING, running(37)); + when(fixture.runner.inFlightSnapshot(OPERATION)).thenReturn(Optional.of(running(0))); + when(fixture.coordinator.status()).thenReturn(new RetainedCutoverStatus( + OPERATION, RetainedCutoverStatus.Phase.EXECUTING)); + + assertThat(fixture.commands.migration(OPERATION).progressPercent()).isEqualTo(37); + } + + @Test + void liveProjectionFailsClosedWhenTheJournalIsMissingOrCorrupt() throws Exception { + Fixture missing = fixture(); + missing.store.create(pending()); + missing.store.compareAndTransition( + OPERATION, MigrationOperationState.PENDING, running()); + when(missing.runner.inFlightSnapshot(OPERATION)).thenReturn(Optional.of(running())); + when(missing.coordinator.status()).thenReturn(new RetainedCutoverStatus( + OPERATION, RetainedCutoverStatus.Phase.EXECUTING)); + Files.delete(root.resolve(FileMigrationOperationStore.RELATIVE_PATH)); + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> missing.commands.migration(OPERATION)); + + Path corruptRoot = root.resolve("corrupt"); + FileMigrationOperationStore corruptStore = new FileMigrationOperationStore(corruptRoot); + DeploymentMigrationCommandRunner runner = mock(DeploymentMigrationCommandRunner.class); + RetainedCutoverCoordinator coordinator = mock(RetainedCutoverCoordinator.class); + ManagedDeploymentMigrationCommands commands = new ManagedDeploymentMigrationCommands( + runner, corruptStore, mock(ManagedMigrationConfigurationTransaction.class), coordinator); + corruptStore.create(pending()); + corruptStore.compareAndTransition( + OPERATION, MigrationOperationState.PENDING, running()); + Files.writeString(corruptRoot.resolve(FileMigrationOperationStore.RELATIVE_PATH), "broken"); + when(runner.inFlightSnapshot(OPERATION)).thenReturn(Optional.of(running())); + when(coordinator.status()).thenReturn(new RetainedCutoverStatus( + OPERATION, RetainedCutoverStatus.Phase.EXECUTING)); + assertStoreError(SetupErrorCode.CONFIG_RECOVERY_REQUIRED, + () -> commands.migration(OPERATION)); + } + @Test void retainedStatusRequiresAnExactReadyJournalShape() { Fixture fixture = fixture(); @@ -266,7 +376,11 @@ class ManagedDeploymentMigrationCommandsTest { } private static MigrationOperationSnapshot running() { - return snapshot(MigrationOperationState.RUNNING, MigrationStage.COPYING, 0, + return running(0); + } + + private static MigrationOperationSnapshot running(int progress) { + return snapshot(MigrationOperationState.RUNNING, MigrationStage.COPYING, progress, VerificationState.PENDING, false, false); } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationCommandFlowTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationCommandFlowTest.java new file mode 100644 index 0000000000..6d13c97e11 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/ManagedMigrationCommandFlowTest.java @@ -0,0 +1,267 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.sql.Connection; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceLease; +import org.apache.hertzbeat.manager.maintenance.MigrationMaintenanceOrchestrator; +import org.apache.hertzbeat.manager.maintenance.MigrationSourceAction; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MetadataMigrationRequest; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationView; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.TargetInspection; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.ApplyMode; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseConfiguration; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.CandidateRef; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.MetadataTargetStageResult; +import org.apache.hertzbeat.manager.setup.config.ManagedMigrationConfigurationTransaction.StageOutcome; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; + +@Timeout(15) +class ManagedMigrationCommandFlowTest { + + private static final String OPERATION = "operation-a"; + private static final String IDENTITY = "a".repeat(64); + private static final MetadataDatabaseConfiguration DATABASE = new MetadataDatabaseConfiguration( + MetadataDatabaseKind.MYSQL, "jdbc:mysql://db.example/hertzbeat", + "migration", "private-password"); + + @TempDir + private Path root; + + @Test + void sameOperationJoinsTheRealRunnerAndCoordinatorDuringCopyAndHandoff() throws Exception { + CountDownLatch copyEntered = new CountDownLatch(1); + CountDownLatch releaseCopy = new CountDownLatch(1); + CountDownLatch finalPublishEntered = new CountDownLatch(1); + CountDownLatch releaseFinalPublish = new CountDownLatch(1); + MigrationOperationFilePublisher delegate = new MigrationOperationFilePublisher(root); + FileMigrationOperationStore store = new FileMigrationOperationStore(root, (target, content) -> { + if (new String(content, StandardCharsets.UTF_8).contains("state=READY_TO_ACTIVATE")) { + finalPublishEntered.countDown(); + await(releaseFinalPublish); + } + delegate.publish(target, content); + }); + ManagedMigrationConfigurationTransaction configuration = configuration(); + TargetJdbcConnectionFactory factory = mock(TargetJdbcConnectionFactory.class); + TargetJdbcConnectionLease provisionLease = lease(IDENTITY); + TargetJdbcConnectionLease copyLease = lease(IDENTITY); + when(factory.acquire(any(), any(), any())).thenReturn(provisionLease, copyLease); + FlywayTargetSchemaProvisioner provisioner = mock(FlywayTargetSchemaProvisioner.class); + when(provisioner.provision(any(), eq(MetadataDatabaseKind.MYSQL), any())) + .thenReturn(new TargetSchemaProvisioningOutcome( + TargetSchemaConnectionDisposition.REUSABLE)); + MigrationMaintenanceOrchestrator maintenance = mock(MigrationMaintenanceOrchestrator.class); + MigrationMaintenanceLease maintenanceLease = sourceLease(); + when(maintenance.acquire(eq(OPERATION), any())).thenReturn(maintenanceLease); + JdbcMetadataMigrationExecutor executor = mock(JdbcMetadataMigrationExecutor.class); + doAnswer(ignored -> { + copyEntered.countDown(); + await(releaseCopy); + return null; + }).when(executor).execute(any(), any(), eq(MetadataDatabaseKind.MYSQL), + any(JdbcMetadataMigrationDeadline.class), any()); + RetainedCutoverCoordinator coordinator = new RetainedCutoverCoordinator( + factory, provisioner, maintenance, executor, System::nanoTime); + + try (DeploymentMigrationCommandRunner runner = new DeploymentMigrationCommandRunner( + store, configuration, coordinator, + Clock.fixed(Instant.parse("2026-08-10T04:00:00Z"), ZoneOffset.UTC), + Duration.ofSeconds(5))) { + ManagedDeploymentMigrationCommands commands = new ManagedDeploymentMigrationCommands( + runner, store, configuration, coordinator); + MetadataMigrationTargetInspector inspector = mock(MetadataMigrationTargetInspector.class); + when(inspector.inspect(any(), any(), any())).thenReturn(TargetInspection.EMPTY); + DeploymentViewProjector projector = mock(DeploymentViewProjector.class); + when(projector.project()).thenReturn(DefaultDeploymentWorkflowTest.deploymentFixture()); + DefaultDeploymentWorkflow workflow = new DefaultDeploymentWorkflow( + projector, commands, inspector, new MetadataMigrationPolicy(), + new DeploymentWorkflowFailureMapper(), + Clock.fixed(Instant.parse("2026-08-10T04:00:00Z"), ZoneOffset.UTC), + Duration.ofSeconds(5), System::nanoTime); + MetadataMigrationRequest request = new MetadataMigrationRequest( + OPERATION, MigrationTarget.MYSQL, DATABASE, ApplyMode.MANAGED_WRITE); + try { + MigrationView first = workflow.migrate(request); + assertThat(copyEntered.await(5, SECONDS)).isTrue(); + assertRunning(first); + assertRunning(workflow.migrate(request)); + assertRunning(workflow.migration(OPERATION)); + assertThat(coordinator.status().phase()) + .isEqualTo(RetainedCutoverStatus.Phase.EXECUTING); + verify(inspector).inspect(any(), any(), any()); + verify(executor).execute(any(), any(), any(), + any(JdbcMetadataMigrationDeadline.class), any()); + + releaseCopy.countDown(); + assertThat(finalPublishEntered.await(5, SECONDS)).isTrue(); + assertThat(coordinator.status().phase()) + .isEqualTo(RetainedCutoverStatus.Phase.HANDOFFING); + assertRunning(workflow.migrate(request)); + assertRunning(workflow.migration(OPERATION)); + verify(inspector).inspect(any(), any(), any()); + verify(executor).execute(any(), any(), any(), + any(JdbcMetadataMigrationDeadline.class), any()); + } finally { + releaseCopy.countDown(); + releaseFinalPublish.countDown(); + } + awaitRetained(coordinator); + verify(executor, times(1)).execute(any(), any(), any(), + any(JdbcMetadataMigrationDeadline.class), any()); + coordinator.releaseRetained(OPERATION); + } finally { + releaseCopy.countDown(); + releaseFinalPublish.countDown(); + } + } + + @Test + void blockedPreparationReplaysWhileExactReleaseCleanupStillOwnsTheWorker() throws Exception { + FileMigrationOperationStore store = new FileMigrationOperationStore(root); + ManagedMigrationConfigurationTransaction configuration = mock( + ManagedMigrationConfigurationTransaction.class); + when(configuration.stageMetadataTarget(any(), any(), any(), any(), any())) + .thenReturn(new MetadataTargetStageResult( + StageOutcome.RECOVERY_REQUIRED, Optional.empty())); + CountDownLatch cleanupEntered = new CountDownLatch(1); + CountDownLatch releaseCleanup = new CountDownLatch(1); + TargetJdbcConnectionLease provisionLease = mock(TargetJdbcConnectionLease.class); + when(provisionLease.targetIdentityHash()).thenReturn(IDENTITY); + doAnswer(ignored -> { + cleanupEntered.countDown(); + await(releaseCleanup); + return null; + }).when(provisionLease).close(); + TargetJdbcConnectionFactory factory = mock(TargetJdbcConnectionFactory.class); + when(factory.acquire(any(), any(), any())).thenReturn(provisionLease); + FlywayTargetSchemaProvisioner provisioner = mock(FlywayTargetSchemaProvisioner.class); + JdbcMetadataMigrationExecutor executor = mock(JdbcMetadataMigrationExecutor.class); + RetainedCutoverCoordinator coordinator = new RetainedCutoverCoordinator( + factory, provisioner, mock(MigrationMaintenanceOrchestrator.class), + executor, System::nanoTime); + + try (DeploymentMigrationCommandRunner runner = new DeploymentMigrationCommandRunner( + store, configuration, coordinator, + Clock.fixed(Instant.parse("2026-08-10T04:00:00Z"), ZoneOffset.UTC), + Duration.ofSeconds(5))) { + ManagedDeploymentMigrationCommands commands = new ManagedDeploymentMigrationCommands( + runner, store, configuration, coordinator); + MetadataMigrationTargetInspector inspector = mock(MetadataMigrationTargetInspector.class); + when(inspector.inspect(any(), any(), any())).thenReturn(TargetInspection.EMPTY); + DeploymentViewProjector projector = mock(DeploymentViewProjector.class); + when(projector.project()).thenReturn(DefaultDeploymentWorkflowTest.deploymentFixture()); + DefaultDeploymentWorkflow workflow = new DefaultDeploymentWorkflow( + projector, commands, inspector, new MetadataMigrationPolicy(), + new DeploymentWorkflowFailureMapper(), Clock.systemUTC(), + Duration.ofSeconds(5), System::nanoTime); + MetadataMigrationRequest request = new MetadataMigrationRequest( + OPERATION, MigrationTarget.MYSQL, DATABASE, ApplyMode.MANAGED_WRITE); + try { + MigrationView first = workflow.migrate(request); + assertThat(cleanupEntered.await(5, SECONDS)).isTrue(); + assertThat(first.state()).isEqualTo(MigrationOperationState.PENDING); + assertThat(first.errorCode()).isEqualTo( + org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode + .CONFIG_RECOVERY_REQUIRED); + + assertThat(workflow.migrate(request)).isEqualTo(first); + verify(inspector).inspect(any(), any(), any()); + verify(factory).acquire(any(), any(), any()); + verify(provisioner, org.mockito.Mockito.never()).provision(any(), any(), any()); + verify(executor, org.mockito.Mockito.never()).execute( + any(), any(), any(), any(JdbcMetadataMigrationDeadline.class), any()); + } finally { + releaseCleanup.countDown(); + } + } finally { + releaseCleanup.countDown(); + } + } + + private ManagedMigrationConfigurationTransaction configuration() throws Exception { + ManagedMigrationConfigurationTransaction configuration = mock( + ManagedMigrationConfigurationTransaction.class); + when(configuration.stageMetadataTarget(any(), any(), any(), any(), any())) + .thenAnswer(invocation -> new MetadataTargetStageResult( + StageOutcome.STAGED, + Optional.of(new CandidateRef( + invocation.getArgument(0), invocation.getArgument(1))))); + return configuration; + } + + private static TargetJdbcConnectionLease lease(String identity) { + TargetJdbcConnectionLease lease = mock(TargetJdbcConnectionLease.class); + when(lease.targetIdentityHash()).thenReturn(identity); + Connection connection = mock(Connection.class); + doAnswer(invocation -> { + TargetJdbcConnectionAction action = invocation.getArgument(0); + action.execute(connection); + return null; + }).when(lease).withConnection(any()); + return lease; + } + + private static MigrationMaintenanceLease sourceLease() { + MigrationMaintenanceLease lease = mock(MigrationMaintenanceLease.class); + Connection source = mock(Connection.class); + doAnswer(invocation -> { + MigrationSourceAction action = invocation.getArgument(0); + action.execute(source); + return null; + }).when(lease).withSourceConnection(any()); + return lease; + } + + private static void assertRunning(MigrationView view) { + assertThat(view.operationId()).isEqualTo(OPERATION); + assertThat(view.state()).isEqualTo(MigrationOperationState.RUNNING); + } + + private static void awaitRetained(RetainedCutoverCoordinator coordinator) { + long deadline = System.nanoTime() + SECONDS.toNanos(5); + while (coordinator.status().phase() != RetainedCutoverStatus.Phase.RETAINED + && System.nanoTime() < deadline) { + Thread.onSpinWait(); + } + assertThat(coordinator.status().phase()).isEqualTo(RetainedCutoverStatus.Phase.RETAINED); + } + + private static void await(CountDownLatch latch) { + try { + assertThat(latch.await(5, SECONDS)).isTrue(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new AssertionError(interrupted); + } + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationTestSnapshots.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationTestSnapshots.java new file mode 100644 index 0000000000..b03b814a23 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/setup/workflow/MigrationTestSnapshots.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0. + */ + +package org.apache.hertzbeat.manager.setup.workflow; + +import java.time.Instant; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationOperationState; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationStage; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationTarget; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.MigrationView; +import org.apache.hertzbeat.manager.setup.api.DeploymentApiContract.VerificationState; +import org.apache.hertzbeat.manager.setup.api.SetupApiContract.MetadataDatabaseKind; + +final class MigrationTestSnapshots { + + private static final Instant CREATED = Instant.parse("2026-08-10T01:00:00Z"); + + private MigrationTestSnapshots() { + } + + static MigrationView running(String operationId) { + return new MigrationView( + operationId, MigrationOperationState.RUNNING, MetadataDatabaseKind.H2, + MigrationTarget.POSTGRESQL, MigrationStage.COPYING, 0, + CREATED, CREATED.plusSeconds(1), null, VerificationState.PENDING, + null, 1_000, false, false, false); + } + + static MigrationView awaitingRestart(String operationId) { + return new MigrationView( + operationId, MigrationOperationState.AWAITING_RESTART, MetadataDatabaseKind.H2, + MigrationTarget.POSTGRESQL, MigrationStage.AWAITING_RESTART, 100, + CREATED, CREATED.plusSeconds(1), null, + VerificationState.SUCCEEDED, null, 250, false, true, false); + } + + static MigrationView blockedPending(String operationId) { + return new MigrationView( + operationId, MigrationOperationState.PENDING, MetadataDatabaseKind.H2, + MigrationTarget.POSTGRESQL, MigrationStage.QUEUED, 0, + CREATED, null, null, VerificationState.PENDING, + org.apache.hertzbeat.manager.setup.api.SetupApiContract.SetupErrorCode + .CONFIG_RECOVERY_REQUIRED, + 0, false, false, false); + } +} From 7d7e396a41d25a25deb357db0dbc11bf248e3206 Mon Sep 17 00:00:00 2001 From: Logic Date: Tue, 11 Aug 2026 00:33:50 +0800 Subject: [PATCH 70/71] Stabilize startup context transitions --- hertzbeat-startup/pom.xml | 4 +- .../startup/HertzBeatApplication.java | 1 + .../runtime/HertzBeatStartupCoordinator.java | 25 +++- .../StartupDatabaseDriverPackagingTest.java | 54 +++++++++ .../HertzBeatStartupProcessLifetimeTest.java | 113 ++++++++++++++++++ 5 files changed, 194 insertions(+), 3 deletions(-) create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/StartupDatabaseDriverPackagingTest.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupProcessLifetimeTest.java diff --git a/hertzbeat-startup/pom.xml b/hertzbeat-startup/pom.xml index 997d785106..4f5efb6d81 100644 --- a/hertzbeat-startup/pom.xml +++ b/hertzbeat-startup/pom.xml @@ -189,12 +189,12 @@ com.mysql mysql-connector-j - test + runtime org.postgresql postgresql - test + runtime diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/HertzBeatApplication.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/HertzBeatApplication.java index 6555a4b79b..1dd0d933e1 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/HertzBeatApplication.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/HertzBeatApplication.java @@ -64,6 +64,7 @@ public class HertzBeatApplication { Runtime.getRuntime().addShutdownHook( Thread.ofPlatform().name("hertzbeat-standalone-owner-close").unstarted(coordinator::close)); coordinator.start(args); + coordinator.awaitTermination(); } @PostConstruct diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java index bd6180bf5f..221bc0834f 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupCoordinator.java @@ -19,6 +19,7 @@ package org.apache.hertzbeat.startup.runtime; import java.time.Duration; import java.util.Objects; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; import org.apache.hertzbeat.common.runtime.RuntimeMode; import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; @@ -35,6 +36,7 @@ public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition private final StartupMigrationRecoveryPreflightFactory preflightFactory; private final Duration migrationRecoveryTimeout; private final Executor abortExecutor; + private final CountDownLatch termination = new CountDownLatch(1); private String[] args = new String[0]; private RunningApplicationContext currentContext; private ResolvedStartupInstallationRoot installationRoot; @@ -161,6 +163,22 @@ public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition return currentContext; } + /** Keeps the standalone process alive while Spring contexts are replaced. */ + public void awaitTermination() { + boolean interrupted = false; + while (true) { + try { + termination.await(); + break; + } catch (InterruptedException ignored) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + private RunningApplicationContext transitionInternal(StartupPlan plan) { StartupDecision decision = plan.decision(); if (migrationPreflight != null) { @@ -307,10 +325,15 @@ public final class HertzBeatStartupCoordinator implements SetupRuntimeTransition @Override public synchronized void close() { if (closed && currentContext == null && migrationPreflight == null && deploymentOwner == null) { + termination.countDown(); return; } closed = true; - StartupCleanup.rethrow(cleanupResources(null)); + Throwable failure = cleanupResources(null); + if (failure == null && currentContext == null && migrationPreflight == null && deploymentOwner == null) { + termination.countDown(); + } + StartupCleanup.rethrow(failure); } private Throwable cleanupResources(Throwable primary) { diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/StartupDatabaseDriverPackagingTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/StartupDatabaseDriverPackagingTest.java new file mode 100644 index 0000000000..1a71e08e82 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/StartupDatabaseDriverPackagingTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.regex.Pattern; +import org.junit.jupiter.api.Test; + +class StartupDatabaseDriverPackagingTest { + + @Test + void supportedManagedDatabaseDriversAreRuntimeDependencies() throws Exception { + String pom = Files.readString(repositoryRoot().resolve("hertzbeat-startup/pom.xml")); + + for (String artifactId : List.of("mysql-connector-j", "postgresql")) { + Pattern runtimeDependency = Pattern.compile("\\s*[^<]+\\s*" + + "" + Pattern.quote(artifactId) + "\\s*" + + "runtime\\s*"); + assertThat(runtimeDependency.matcher(pom).find()) + .as(artifactId + " must be packaged for post-setup Spring contexts") + .isTrue(); + } + } + + private static Path repositoryRoot() { + Path current = Path.of("").toAbsolutePath(); + while (current != null && !Files.exists(current.resolve("hertzbeat-startup/pom.xml"))) { + current = current.getParent(); + } + if (current == null) { + throw new IllegalStateException("repository root not found"); + } + return current; + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupProcessLifetimeTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupProcessLifetimeTest.java new file mode 100644 index 0000000000..1e14165f5c --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/runtime/HertzBeatStartupProcessLifetimeTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup.runtime; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Duration; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hertzbeat.common.runtime.RuntimeMode; +import org.apache.hertzbeat.manager.setup.runtime.SetupRuntimeTransition; +import org.junit.jupiter.api.Test; + +class HertzBeatStartupProcessLifetimeTest { + + @Test + void processLifetimeRemainsOwnedAcrossContextReplacementUntilCoordinatorClose() throws Exception { + AtomicReference transition = new AtomicReference<>(); + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( + ignored -> new StartupDecision(RuntimeMode.FULL_SETUP_GATED), + (decision, args, runtimeTransition) -> { + transition.set(runtimeTransition); + return context(decision.mode()); + }); + coordinator.start(new String[0]); + Thread waiter = Thread.ofPlatform().name("startup-process-lifetime-test") + .start(coordinator::awaitTermination); + try { + awaitWaiting(waiter); + + transition.get().completeSetup(); + + assertTrue(waiter.isAlive()); + coordinator.close(); + waiter.join(Duration.ofSeconds(2)); + assertFalse(waiter.isAlive()); + } finally { + coordinator.close(); + waiter.join(Duration.ofSeconds(2)); + } + } + + @Test + void processLifetimeWaitRestoresInterruptOnlyAfterCoordinatorClose() throws Exception { + HertzBeatStartupCoordinator coordinator = new HertzBeatStartupCoordinator( + ignored -> new StartupDecision(RuntimeMode.FULL_SETUP_GATED), + (decision, args, runtimeTransition) -> context(decision.mode())); + coordinator.start(new String[0]); + AtomicBoolean interrupted = new AtomicBoolean(); + Thread waiter = Thread.ofPlatform().name("startup-process-lifetime-interrupt-test").start(() -> { + Thread.currentThread().interrupt(); + coordinator.awaitTermination(); + interrupted.set(Thread.currentThread().isInterrupted()); + }); + try { + awaitWaiting(waiter); + + assertTrue(waiter.isAlive()); + coordinator.close(); + waiter.join(Duration.ofSeconds(2)); + assertFalse(waiter.isAlive()); + assertTrue(interrupted.get()); + } finally { + coordinator.close(); + waiter.join(Duration.ofSeconds(2)); + } + } + + private static RunningApplicationContext context(RuntimeMode mode) { + return new RunningApplicationContext() { + private boolean active = true; + + @Override + public RuntimeMode mode() { + return mode; + } + + @Override + public boolean isActive() { + return active; + } + + @Override + public void close() { + active = false; + } + }; + } + + private static void awaitWaiting(Thread thread) throws InterruptedException { + long deadline = System.nanoTime() + Duration.ofSeconds(2).toNanos(); + while (thread.getState() != Thread.State.WAITING && System.nanoTime() < deadline) { + Thread.sleep(1); + } + assertTrue(thread.getState() == Thread.State.WAITING, "process lifetime waiter must block"); + } +} From 1eb35646d7896bd628bef3e7b952895fb535295f Mon Sep 17 00:00:00 2001 From: Logic Date: Tue, 11 Aug 2026 00:36:03 +0800 Subject: [PATCH 71/71] Align startup release readiness contract --- .../ReleaseReadinessRuntimeAssemblyTest.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/ReleaseReadinessRuntimeAssemblyTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/ReleaseReadinessRuntimeAssemblyTest.java index 84278eabd5..552c9f982e 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/ReleaseReadinessRuntimeAssemblyTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/ReleaseReadinessRuntimeAssemblyTest.java @@ -349,20 +349,20 @@ class ReleaseReadinessRuntimeAssemblyTest { } @Test - void signalWorkspaceTablesStayInV200BaselineOnly() throws IOException { + void signalWorkspaceTablesAreCreatedOnlyByV200OrCurrentBaselines() throws IOException { Path migrationRoot = repoRoot().resolve("hertzbeat-startup/src/main/resources/db/migration"); try (Stream migrationFiles = Files.walk(migrationRoot)) { List laterSignalMigrations = migrationFiles .filter(Files::isRegularFile) - .filter(path -> path.getFileName().toString().endsWith(".sql")) + .filter(path -> path.getFileName().toString().startsWith("V")) .filter(path -> !path.getFileName().toString().startsWith("V200__create_entity_foundation")) - .filter(path -> containsSignalWorkspaceTable(path)) + .filter(path -> createsSignalWorkspaceTable(path)) .map(migrationRoot::relativize) .map(Path::toString) .toList(); assertThat(laterSignalMigrations) - .as("signal workspace tables must stay in the V200 baseline instead of a later V213-style migration") + .as("versioned migrations after V200 may evolve but must not recreate signal workspace tables") .isEmpty(); } } @@ -454,12 +454,12 @@ class ReleaseReadinessRuntimeAssemblyTest { .contains(rule); } - private static boolean containsSignalWorkspaceTable(Path path) { + private static boolean createsSignalWorkspaceTable(Path path) { try { String content = Files.readString(path).toLowerCase(); - return content.contains("hzb_signal_saved_view") - || content.contains("hzb_signal_dashboard_panel_draft") - || content.contains("hzb_signal_dashboard"); + return content.contains("create table hzb_signal_saved_view") + || content.contains("create table hzb_signal_dashboard_panel_draft") + || content.contains("create table hzb_signal_dashboard"); } catch (IOException e) { throw new IllegalStateException("Failed to read migration " + path, e); }