Migrations triggered by Maintenance (system) not reported on VM events list

engine/schema/src/main/java/com/cloud/event/dao/EventDao.java

@@ -19,6 +19,7 @@
 import java.util.Date;
 import java.util.List;
 
+import com.cloud.event.Event;
 import com.cloud.event.EventVO;
 import com.cloud.utils.db.Filter;
 import com.cloud.utils.db.GenericDao;
@@ -31,6 +32,18 @@ public interface EventDao extends GenericDao<EventVO, Long> {
 
     EventVO findCompletedEvent(long startId);
 
+    /**
+     * Finds the last non-archived start event matching the specified criteria.
+     * Events are ordered by ID in descending order, returning the most recent one.
+     *
+     * @param type         the event type to search for
+     * @param state        the event state to search for (e.g., {@link Event.State#Scheduled})
+     * @param resourceId   the resource ID associated with the event
+     * @param resourceType the resource type associated with the event
+     * @return the most recent EventVO matching the criteria, or null if not found
+     */
+    EventVO findLastEvent(String type, Event.State state, Long resourceId, String resourceType);
+
     public List<EventVO> listToArchiveOrDeleteEvents(List<Long> ids, String type, Date startDate, Date endDate, List<Long> accountIds);
 
     public void archiveEvents(List<EventVO> events);

engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java

@@ -19,7 +19,6 @@
 import java.util.Date;
 import java.util.List;
 
-
 import org.springframework.stereotype.Component;
 
 import com.cloud.event.Event.State;
@@ -35,6 +34,7 @@
 public class EventDaoImpl extends GenericDaoBase<EventVO, Long> implements EventDao {
     protected final SearchBuilder<EventVO> CompletedEventSearch;
     protected final SearchBuilder<EventVO> ToArchiveOrDeleteEventSearch;
+    protected final SearchBuilder<EventVO> LastStartEventSearch;
 
     public EventDaoImpl() {
         CompletedEventSearch = createSearchBuilder();
@@ -51,6 +51,14 @@ public EventDaoImpl() {
         ToArchiveOrDeleteEventSearch.and("createdDateL", ToArchiveOrDeleteEventSearch.entity().getCreateDate(), Op.LTEQ);
         ToArchiveOrDeleteEventSearch.and("archived", ToArchiveOrDeleteEventSearch.entity().getArchived(), Op.EQ);
         ToArchiveOrDeleteEventSearch.done();
+
+        LastStartEventSearch = createSearchBuilder();
+        LastStartEventSearch.and("type", LastStartEventSearch.entity().getType(), Op.EQ);
+        LastStartEventSearch.and("state", LastStartEventSearch.entity().getState(), Op.EQ);
+        LastStartEventSearch.and("resourceId", LastStartEventSearch.entity().getResourceId(), Op.EQ);
+        LastStartEventSearch.and("resourceType", LastStartEventSearch.entity().getResourceType(), Op.EQ);
+        LastStartEventSearch.and("archived", LastStartEventSearch.entity().getArchived(), Op.EQ);
+        LastStartEventSearch.done();
     }
 
     @Override
@@ -77,6 +85,17 @@ public EventVO findCompletedEvent(long startId) {
         return findOneIncludingRemovedBy(sc);
     }
 
+    @Override
+    public EventVO findLastEvent(String type, State state, Long resourceId, String resourceType) {
+        SearchCriteria<EventVO> sc = LastStartEventSearch.create();
+        sc.setParameters("type", type);
+        sc.setParameters("state", state);
+        sc.setParameters("resourceId", resourceId);
+        sc.setParameters("resourceType", resourceType);
+        sc.setParameters("archived", false);
+        return findLastOneBy(sc);
+    }
+
     @Override
     public List<EventVO> listToArchiveOrDeleteEvents(List<Long> ids, String type, Date startDate, Date endDate, List<Long> accountIds) {
         SearchCriteria<EventVO> sc = ToArchiveOrDeleteEventSearch.create();

framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java

@@ -929,7 +929,7 @@ public Class<T> getEntityBeanType() {
     protected T findOneIncludingRemovedBy(final SearchCriteria<T> sc) {
         Filter filter = new Filter(1, true);
         List<T> results = searchIncludingRemoved(sc, filter, null, false);
-        assert results.size() <= 1 : "Didn't the limiting worked?";
+        assert results.size() <= 1 : "Didn't the limiting work?";
         return results.size() == 0 ? null : results.get(0);
     }
 
@@ -949,6 +949,15 @@ public T findOneBy(SearchCriteria<T> sc, final Filter filter) {
         return results.isEmpty() ? null : results.get(0);
     }
 
+    @DB()
+    protected T findLastOneBy(SearchCriteria<T> sc) {
+        sc = checkAndSetRemovedIsNull(sc);
+        Filter filter = new Filter(_entityBeanType, "id", Boolean.FALSE, 0L, 1L);
+        List<T> results = searchIncludingRemoved(sc, filter, null, false);
+        assert results.size() <= 1 : "Didn't the limiting work?";
+        return results.size() == 0 ? null : results.get(0);
+    }
+
     @DB()
     public List<T> listBy(SearchCriteria<T> sc, final Filter filter) {
         sc = checkAndSetRemovedIsNull(sc);

server/src/main/java/com/cloud/event/ActionEventUtils.java

@@ -400,6 +400,20 @@ private static long getDomainId(long accountId) {
         return account.getDomainId();
     }
 
+    /**
+     * Retrieves the last non-archived event matching the specified criteria.
+     *
+     * @param type         the event type to search for
+     * @param state        the event state to search for (e.g., {@link Event.State#Scheduled})
+     * @param resourceId   the resource ID associated with the event
+     * @param resourceType the resource type associated with the event
+     * @return the most recent EventVO matching the criteria, or null if not found
+     * @see EventDao#findLastEvent(String, Event.State, Long, String)
+     */
+    public static EventVO getLastEvent(String type, Event.State state, Long resourceId, String resourceType) {
+        return s_eventDao.findLastEvent(type, state, resourceId, resourceType);
+    }
+
     private static void populateFirstClassEntities(Map<String, String> eventDescription){
 
         CallContext context = CallContext.current();

server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java

@@ -17,6 +17,7 @@
 package com.cloud.ha;
 
 import static org.apache.cloudstack.framework.config.ConfigKey.Scope.Zone;
+import static com.cloud.event.Event.State;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -32,8 +33,14 @@
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
-import org.apache.cloudstack.api.ApiCommandResourceType;
 import org.apache.cloudstack.context.CallContext;
+import com.cloud.event.ActionEventUtils;
+import com.cloud.event.Event;
+import com.cloud.event.EventTypes;
+import com.cloud.event.EventVO;
+import com.cloud.user.Account;
+import com.cloud.user.User;
+import org.apache.cloudstack.api.ApiCommandResourceType;
 import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider;
@@ -452,9 +459,16 @@ public boolean scheduleMigration(final VMInstanceVO vm, HighAvailabilityManager.
         }
 
         Long hostId = VirtualMachine.State.Migrating.equals(vm.getState()) ? vm.getLastHostId() : vm.getHostId();
-        final HaWorkVO work = new HaWorkVO(vm.getId(), vm.getType(), WorkType.Migration, Step.Scheduled, vm.getHostId(), vm.getState(), 0, vm.getUpdated(), reasonType);
+        final HaWorkVO work = new HaWorkVO(vm.getId(), vm.getType(), WorkType.Migration, Step.Scheduled, hostId, vm.getState(), 0, vm.getUpdated(), reasonType);
         _haDao.persist(work);
-        logger.info("Scheduled migration work of VM {} from host {} with HAWork {}", vm, _hostDao.findById(vm.getHostId()), work);
+
+        HostVO host = _hostDao.findById(hostId);
+        logger.info(String.format("Scheduled migration work of VM %s from host %s with HAWork %s", vm, host, work));
+        String hostName = Optional.ofNullable(host).map(HostVO::getName).orElse("N/A");
+        String msg = String.format("Scheduled migration work of VM %s from host %s (%s) with HAWork %s (attempt %s of %s)",
+                vm.getHostName(), hostId, hostName, work.getId(), work.getTimesTried() + 1, _maxRetries);
+        createEvent(vm.getId(), ApiCommandResourceType.VirtualMachine, EventTypes.EVENT_VM_MIGRATE, msg,
+                State.Scheduled, EventVO.LEVEL_INFO);
         wakeupWorkers();
         return true;
     }
@@ -862,18 +876,77 @@ protected boolean checkAndCancelWorkIfNeeded(final HaWorkVO work) {
         return true;
     }
 
+    /**
+     * Creates an event for {@link ApiCommandResourceType} operations.
+     * This is a fail-safe helper method for logging purposes - exceptions are caught and logged.
+     *
+     * @param resourceId   the resource ID
+     * @param resourceType the event resource type ({@link ApiCommandResourceType})
+     * @param type         the event type ({@link EventTypes})
+     * @param description  the event description
+     * @param state        the event state ({@link Event.State})
+     * @param level        the event level (e.g., {@link EventVO#LEVEL_INFO} or {@link EventVO#LEVEL_ERROR})
+     */
+    private void createEvent(Long resourceId, ApiCommandResourceType resourceType, String type, String description,
+                             State state, String level) {
+        try {
+            String resourceTypeStr = resourceType.toString();
+            Long userId = User.UID_SYSTEM;
+            Long accountId = Account.ACCOUNT_ID_SYSTEM;
+            if (ApiCommandResourceType.VirtualMachine.equals(resourceType) && resourceId != null) {
+                VMInstanceVO vm = _instanceDao.findById(resourceId);
+                if (vm != null) {
+                    accountId = vm.getAccountId();
+                }
+            }
+            long startEventId = state == State.Scheduled ? 0L
+                    : Optional.ofNullable(ActionEventUtils.getLastEvent(type, State.Scheduled, resourceId,
+                            resourceTypeStr))
+                    .map(EventVO::getId).orElse(0L);
+
+            switch (state) {
+                case Started:
+                    ActionEventUtils.onStartedActionEvent(userId, accountId, type, description, resourceId,
+                            resourceTypeStr, true, startEventId);
+                    break;
+                case Scheduled:
+                    ActionEventUtils.onScheduledActionEvent(userId, accountId, type, description, resourceId,
+                            resourceTypeStr, true, startEventId);
+                    break;
+                case Completed:
+                    ActionEventUtils.onCompletedActionEvent(userId, accountId, level, type, true,
+                            description, resourceId, resourceTypeStr, startEventId);
+                    break;
+                default:
+                    throw new CloudRuntimeException("Unsupported event state: " + state);
+            }
+        } catch (Exception e) {
+            logger.error(String.format("Failed to create event for VM: %s, command: %s, state: %s, level: %s",
+                    resourceId, type, state, level), e);
+        }
+    }
+
     public Long migrate(final HaWorkVO work) {
         logger.debug("MIGRATE with HA WORK");
         long vmId = work.getInstanceId();
         long srcHostId = work.getHostId();
         HostVO srcHost = _hostDao.findById(srcHostId);
+        ApiCommandResourceType resourceType = ApiCommandResourceType.VirtualMachine;
+        String eventType = EventTypes.EVENT_VM_MIGRATE;
+        int attemptNumber = work.getTimesTried() + 1;
 
         VMInstanceVO vm = _instanceDao.findById(vmId);
         if (vm == null) {
-            logger.info("Unable to find vm: {}, skipping migrate.", vmId);
+            String msg = String.format("Unable to find vm %s, skipping migration. HA Work %s (attempt %s of %s)",
+                    vmId, work.getId(), attemptNumber, _maxRetries);
+            logger.info(msg);
+            createEvent(vmId, resourceType, eventType, msg, State.Completed, EventVO.LEVEL_ERROR);
             return null;
         }
         if (checkAndCancelWorkIfNeeded(work)) {
+            String msg = String.format("Cancelled migration for vm %s as it is not needed anymore. HA Work %s (attempt %s of %s)",
+                    vm.getHostName(), work.getId(), attemptNumber, _maxRetries);
+            createEvent(vmId, resourceType, eventType, msg, State.Completed, EventVO.LEVEL_ERROR);
             return null;
         }
         logger.info("Migration attempt: for {} from {}. Starting attempt: {}/{} times.", vm, srcHost, 1 + work.getTimesTried(), _maxRetries);
@@ -883,23 +956,41 @@ public Long migrate(final HaWorkVO work) {
             return null;
         }
         if (VirtualMachine.State.Running.equals(vm.getState()) && srcHostId != vm.getHostId()) {
-            logger.info("VM {} is running on a different host {}, skipping migration", vm, vm.getHostId());
+            String vmHostName = Optional.ofNullable(_hostDao.findById(vm.getHostId())).map(HostVO::getName)
+                    .orElse("N/A");
+            String msg = String.format("VM %s is running on a different host (%s), skipping migration. HA Work %s (attempt %s of %s)",
+                    vm.getHostName(), vmHostName, work.getId(), attemptNumber, _maxRetries);
+            logger.info(msg);
+            createEvent(vmId, resourceType, eventType, msg, State.Completed, EventVO.LEVEL_ERROR);
             return null;
         }
-
+        logger.info(String.format("Migration attempt: for VM %s from host %s. Starting attempt: %d/%d times.",
+                vm, srcHost, attemptNumber, _maxRetries));
         try {
+            String vmHostName = Optional.ofNullable(_hostDao.findById(vm.getHostId())).map(HostVO::getName)
+                    .orElse("N/A");
+            String msg = String.format("Starting migration from host %s. HA Work %s (attempt %s of %s)",
+                    vmHostName, work.getId(), attemptNumber, _maxRetries);
+            createEvent(vmId, resourceType, eventType, msg, State.Started, EventVO.LEVEL_INFO);
             work.setStep(Step.Migrating);
             _haDao.update(work.getId(), work);
-
             // First try starting the vm with its original planner, if it doesn't succeed send HAPlanner as its an emergency.
             _itMgr.migrateAway(vm.getUuid(), srcHostId);
+            msg = String.format("Completed migration. HA Work %s (attempt %s of %s)", work.getId(), attemptNumber, _maxRetries);
+            createEvent(vmId, resourceType, eventType, msg, State.Completed, EventVO.LEVEL_INFO);
             return null;
         } catch (InsufficientServerCapacityException e) {
-            logger.warn("Migration attempt: Insufficient capacity for migrating a VM {} from source host {}. Exception: {}", vm, srcHost, e.getMessage());
+            String msg = String.format("Migration attempt: Insufficient capacity for migrating a VM %s from source host %s. HA Work %s (attempt %s of %s)",
+                    vm.getHostName(), srcHost, work.getId(), attemptNumber, _maxRetries);
+            logger.warn(msg);
             _resourceMgr.migrateAwayFailed(srcHostId, vmId);
+            createEvent(vmId, resourceType, eventType, msg, State.Completed, EventVO.LEVEL_ERROR);
             return (System.currentTimeMillis() >> 10) + _migrateRetryInterval;
         } catch (Exception e) {
-            logger.warn("Migration attempt: Unexpected exception occurred when attempting migration of {} {}", vm, e.getMessage());
+            String msg = String.format("Migration attempt: Unexpected exception occurred when attempting migration of vm %s. HA Work %s (attempt %s of %s)",
+                    vm.getHostName(), work.getId(), attemptNumber, _maxRetries);
+            logger.warn(msg);
+            createEvent(vmId, resourceType, eventType, msg, State.Completed, EventVO.LEVEL_ERROR);
             throw e;
         }
     }

server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java

@@ -78,6 +78,7 @@
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachineManager;
 import com.cloud.vm.dao.VMInstanceDao;
+import org.springframework.test.util.ReflectionTestUtils;
 
 @RunWith(MockitoJUnitRunner.class)
 public class HighAvailabilityManagerImplTest {
@@ -309,7 +310,12 @@ public void scheduleMigration() {
         Mockito.when(vm.getType()).thenReturn(VirtualMachine.Type.User);
         Mockito.when(vm.getState()).thenReturn(VirtualMachine.State.Running);
         Mockito.when(vm.getHostId()).thenReturn(1L);
-        Mockito.when(_haDao.persist((HaWorkVO)Mockito.any())).thenReturn(Mockito.mock(HaWorkVO.class));
+
+        Mockito.when(_haDao.persist((HaWorkVO) Mockito.any())).thenAnswer(invocation -> {
+            HaWorkVO haWork = invocation.getArgument(0);
+            ReflectionTestUtils.setField(haWork, "id", 1L);
+            return haWork;
+        });
 
         ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
         highAvailabilityManager.VmHaEnabled = haEnabled;