Skip to content

Rewrite of email alert about VR health checks #14403

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
erikbocks wants to merge 1 commit into
base: main
Choose a base branch
from
+25 −17
Open

Rewrite of email alert about VR health checks #14403

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion engine/schema/src/main/java/com/cloud/network/dao/RouterHealthCheckResultDaoImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ public RouterHealthCheckResultVO getRouterHealthCheckResult(long routerId, Strin
SearchCriteria<RouterHealthCheckResultVO> sc = RouterChecksSearchBuilder.create();
sc.setParameters("routerId", routerId);
sc.setParameters("checkName", checkName);
sc.setParameters("checkType", checkType);
sc.setParametersIfNotNull("checkType", checkType);
List<RouterHealthCheckResultVO> checks = listBy(sc);
if (checks.size() > 1) {
logger.error("Found multiple entries for router Id: " + routerId + ", check name: " + checkName);
Expand Down
40 changes: 24 additions & 16 deletions server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@
import com.google.gson.reflect.TypeToken;

import org.apache.cloudstack.acl.ApiKeyPairVO;
import org.apache.cloudstack.alert.AlertService;
import org.apache.cloudstack.alert.AlertService.AlertType;
import org.apache.cloudstack.api.ApiCommandResourceType;
import org.apache.cloudstack.api.command.admin.router.RebootRouterCmd;
Expand Down Expand Up @@ -861,7 +860,7 @@ protected void updateSite2SiteVpnConnectionState(final List<DomainRouterVO> rout
"Site-to-site Vpn Connection to %s on router %s(%s) just switched from %s to %s",
gw.getName(), router.getHostName(), router, oldState, conn.getState());
logger.info(context);
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), title, context);
_alertMgr.sendAlert(AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), title, context);
}
}
} finally {
Expand Down Expand Up @@ -924,7 +923,7 @@ protected void updateRoutersRedundantState(final List<DomainRouterVO> routers) {
router, router.getHostName(), prevState, currState);
logger.info(context);
if (currState == RedundantState.PRIMARY) {
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), title, context);
_alertMgr.sendAlert(AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), title, context);
}
}
}
Expand Down Expand Up @@ -1002,7 +1001,7 @@ private void checkDuplicatePrimary(final List<DomainRouterVO> routers) {
"with hostname %s are both in PRIMARY state! " +
"If the problem persist, restart both of routers. ",
router, router.getHostName(), dupRouter, dupRouter.getHostName());
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), title, context);
_alertMgr.sendAlert(AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), title, context);
logger.warn(context);
} else {
networkRouterMaps.put(routerGuestNtwkId, router);
Expand Down Expand Up @@ -1160,23 +1159,33 @@ private void handleFailingChecks(DomainRouterVO router, List<String> failingChec
if (CollectionUtils.isEmpty(failingChecks)) {
return;
}
String subject = String.format("Failed health checks on router [%s]", router.getName());
String alertMessage = String.format("The following health checks have failed on router [%s] with UUID [%s]: ", router.getName(), router.getUuid());
ArrayList<String> failedChecks = new ArrayList<>();
for (String failedCheckName : failingChecks) {
RouterHealthCheckResultVO routerHealthCheckResultVO = routerHealthCheckResultDao.getRouterHealthCheckResult(router.getId(), failedCheckName, null);
String failedCheckDetails = routerHealthCheckResultVO.getParsedCheckDetails();

@Damans227 Damans227 Oct 14, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what happens if there is no saved result for a failing check? looks like it crashes here and the router recreate and the other routers get skipped

failedCheckDetails = failedCheckDetails.replace("\n", " ");
String failedCheck = failedCheckName + ": " + failedCheckDetails;
failedChecks.add(failedCheck);
}
alertMessage = alertMessage + failedChecks;

String alertMessage = String.format("Health checks failed: %d failing checks on router %s / %s", failingChecks.size(), router.getName(), router.getUuid());
_alertMgr.sendAlert(AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(),
alertMessage, alertMessage);
logger.warn(alertMessage + ". Checking failed health checks to see if router needs recreate");
subject, alertMessage);
logger.warn("{}. Checking failed health checks to see if router needs recreation", alertMessage);

String checkFailsToRecreateVr = RouterHealthChecksFailuresToRecreateVr.valueIn(router.getDataCenterId());
StringBuilder failingChecksEvent = new StringBuilder();
boolean recreateRouter = false;
for (int i = 0; i < failingChecks.size(); i++) {
String failedCheck = failingChecks.get(i);
if (i == 0) {
failingChecksEvent.append("Router ")
failingChecksEvent.append("Router [")
.append(router.getName())
.append(" / ")
.append("] with UUID [")
.append(router.getUuid())
.append(" has failing checks: ");
.append("] has failing checks: ");
}

failingChecksEvent.append(failedCheck);
Expand All @@ -1193,8 +1202,7 @@ private void handleFailingChecks(DomainRouterVO router, List<String> failingChec
Domain.ROOT_DOMAIN, EventTypes.EVENT_ROUTER_HEALTH_CHECKS, failingChecksEvent.toString(), router.getId(), ApiCommandResourceType.DomainRouter.toString());

if (recreateRouter) {
logger.warn("Health Check Alert: Found failing checks in " +
RouterHealthChecksFailuresToRecreateVrCK + ", attempting recreating router.");
logger.warn("Health Check Alert: Found failing checks in [{}], attempting to recreate router with id [{}].", RouterHealthChecksFailuresToRecreateVrCK, router.getId());
recreateRouter(router);
}
}
Expand Down Expand Up @@ -2303,7 +2311,7 @@ private Provider getVrProvider(DomainRouterVO router) {
if (vrProvider == null) {
throw new CloudRuntimeException("Cannot find related virtual router provider of router: " + router.getHostName());
}
final Provider provider = Network.Provider.getProvider(vrProvider.getType().toString());
final Provider provider = Provider.getProvider(vrProvider.getType().toString());
if (provider == null) {
throw new CloudRuntimeException("Cannot find related provider of virtual router provider: " + vrProvider.getType().toString());
}
Expand Down Expand Up @@ -2713,7 +2721,7 @@ protected void finalizeIpAssocForNetwork(final Commands cmds, final VirtualRoute
}

protected ArrayList<? extends PublicIpAddress> getPublicIpsToApply(final Provider provider, final Long guestNetworkId,
final com.cloud.network.IpAddress.State... skipInStates) {
final IpAddress.State... skipInStates) {

final List<? extends IpAddress> userIps = _networkModel.listPublicIpsAssignedToGuestNtwk(guestNetworkId, null);

Expand Down Expand Up @@ -2761,7 +2769,7 @@ public boolean finalizeStart(final VirtualMachineProfile profile, final long hos
final String errorMessage = "Command: " + cmdClassName + " failed while starting virtual router";
final String errorDetails = "Details: " + answer.getDetails() + " " + answer;
// add alerts for the failed commands
_alertMgr.sendAlert(AlertService.AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), errorMessage, errorDetails);
_alertMgr.sendAlert(AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), errorMessage, errorDetails);
logger.error(answer.getDetails());
logger.warn(errorMessage);
// Stop the router if any of the commands failed
Expand Down Expand Up @@ -3180,7 +3188,7 @@ public <T extends VirtualRouter> void collectNetworkStatistics(final T router, f
continue;
}
if (forVpc && network.getTrafficType() == TrafficType.Public || !forVpc && network.getTrafficType() == TrafficType.Guest
&& network.getGuestType() == Network.GuestType.Isolated) {
&& network.getGuestType() == GuestType.Isolated) {
final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), forVpc, routerNic.getIPv4Address());
final String routerType = router.getType().toString();
final UserStatisticsVO previousStats = _userStatsDao.findBy(router.getAccountId(), router.getDataCenterId(), network.getId(),
Expand Down
Loading