From 1240a7e2f7ff0d57a4e772804e676841d209d3a6 Mon Sep 17 00:00:00 2001 From: waterWang Date: Thu, 20 Aug 2026 14:49:17 +0800 Subject: [PATCH] fix: bound the usage event rewind to prevent unbounded re-aggregation UsageManagerImpl.parse() rewinds the aggregation start date to the oldest unprocessed event, but has no bound on how far back it can go. If an event cannot be successfully processed (e.g. references a removed entity), the rewind pins the window to that event's date permanently. Each subsequent run re-aggregates from that date to the present, growing by one aggregation period per run. This causes unbounded growth of cloud_usage (54M+ rows reported) and exec_time (42+ minutes per hour). Fix: bound the rewind to 24 hours. The rewind exists to absorb clock skew between the cloud and usage databases, not to replay history. Events older than 24 hours from the current window start will still be retried, but the aggregation window will not be rewound to them. Fixes #13906 Signed-off-by: waterWang --- usage/src/main/java/com/cloud/usage/UsageManagerImpl.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java index eab371ab353f..48d14b53cc21 100644 --- a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java @@ -103,6 +103,7 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna private static final int HOURLY_TIME = 60; private static final int DAILY_TIME = 60 * 24; private static final int THREE_DAYS_IN_MINUTES = 60 * 24 * 3; + private static final long MAX_EVENT_REWIND_MILLIS = 24L * 60 * 60 * 1000; @Inject private AccountDao _accountDao; @@ -699,7 +700,10 @@ public void parse(UsageJobVO job, long startDateMillis, long endDateMillis) { if ((events != null) && (events.size() > 0)) { Date oldestEventDate = events.get(0).getCreateDate(); if (oldestEventDate.getTime() < startDateMillis) { - startDateMillis = oldestEventDate.getTime(); + // Bound the rewind so a single un-processable event cannot pin the + // aggregation window to an arbitrarily old date and cause unbounded + // re-aggregation of the entire history on every run. + startDateMillis = Math.max(oldestEventDate.getTime(), startDateMillis - MAX_EVENT_REWIND_MILLIS); startDate = new Date(startDateMillis); }