From 080ab64b0b86af8cf5403df4218c941c2a4265b5 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Oct 2026 18:41:10 +0000 Subject: [PATCH 1/4] Add timeout to Holmes slackbot status request The request had no timeout and runs while holding the Slack sink's grouping lock, so a hung API call blocked all alert workers and alerts were dropped once the queue filled. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013ZNDw6Lj1zMhS5s9mMHgPS --- src/robusta/core/sinks/robusta/robusta_sink.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/robusta/core/sinks/robusta/robusta_sink.py b/src/robusta/core/sinks/robusta/robusta_sink.py index 68ccdc52c..9aecf3996 100644 --- a/src/robusta/core/sinks/robusta/robusta_sink.py +++ b/src/robusta/core/sinks/robusta/robusta_sink.py @@ -824,7 +824,8 @@ def is_holmes_slackbot_connected(self) -> bool: response = requests.post( RUNNER_GET_HOLMES_SLACKBOT_INFO, json=message_json, - headers={"Content-Type": "application/json"} + headers={"Content-Type": "application/json"}, + timeout=10, ) response.raise_for_status() is_connected = bool(response.json().get("integrations")) From ec4f021328e0181b0a8b5b2b8fb59273675680ac Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Oct 2026 19:15:43 +0000 Subject: [PATCH 2/4] Don't let a Supabase error block Slack messages get_session_token ran outside the try, so a Supabase error or timeout raised out of the Slack sender and the alert was never posted. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013ZNDw6Lj1zMhS5s9mMHgPS --- src/robusta/core/sinks/robusta/robusta_sink.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/robusta/core/sinks/robusta/robusta_sink.py b/src/robusta/core/sinks/robusta/robusta_sink.py index 9aecf3996..668491102 100644 --- a/src/robusta/core/sinks/robusta/robusta_sink.py +++ b/src/robusta/core/sinks/robusta/robusta_sink.py @@ -815,10 +815,9 @@ def __update_job(self, new_job: Job, operation: K8sOperationType): def is_holmes_slackbot_connected(self) -> bool: if 'status' in _holmes_slackbot_cache: return _holmes_slackbot_cache['status'] - session_token = self.dal.get_session_token() try: message_json = { - "session_token": session_token, + "session_token": self.dal.get_session_token(), "account_id": self.account_id, } response = requests.post( From 5e2ddf72d84579c744247445e6f0d6ec9cca48a3 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Oct 2026 19:40:15 +0000 Subject: [PATCH 3/4] Cap the Holmes slackbot check at 10s wall-clock requests' timeout only bounds connect and each idle read, so a trickling response, redirects, slow DNS, several unreachable addresses or a slow Supabase session token could still hold the Slack lock far longer. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013ZNDw6Lj1zMhS5s9mMHgPS --- src/robusta/core/sinks/robusta/robusta_sink.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/robusta/core/sinks/robusta/robusta_sink.py b/src/robusta/core/sinks/robusta/robusta_sink.py index 668491102..d0f83e550 100644 --- a/src/robusta/core/sinks/robusta/robusta_sink.py +++ b/src/robusta/core/sinks/robusta/robusta_sink.py @@ -51,6 +51,7 @@ RUNNER_GET_HOLMES_SLACKBOT_INFO = f"{ROBUSTA_API_ENDPOINT}/api/holmes/integrations/slack/runner" HOLMES_SLACKBOT_CACHE_TTL = int(os.getenv("HOLMES_SLACKBOT_CACHE_TTL", 15 * 60)) +HOLMES_SLACKBOT_TIMEOUT_SECONDS = 10 # Define the cache with a single slot and the configured TTL _holmes_slackbot_cache = TTLCache(maxsize=1, ttl=HOLMES_SLACKBOT_CACHE_TTL) @@ -815,6 +816,14 @@ def __update_job(self, new_job: Job, operation: K8sOperationType): def is_holmes_slackbot_connected(self) -> bool: if 'status' in _holmes_slackbot_cache: return _holmes_slackbot_cache['status'] + # Called under the Slack grouping lock; never block alerts. + _holmes_slackbot_cache['status'] = False + status_check = threading.Thread(target=self.__update_holmes_slackbot_status, daemon=True) + status_check.start() + status_check.join(timeout=HOLMES_SLACKBOT_TIMEOUT_SECONDS) + return _holmes_slackbot_cache.get('status', False) + + def __update_holmes_slackbot_status(self): try: message_json = { "session_token": self.dal.get_session_token(), @@ -824,13 +833,9 @@ def is_holmes_slackbot_connected(self) -> bool: RUNNER_GET_HOLMES_SLACKBOT_INFO, json=message_json, headers={"Content-Type": "application/json"}, - timeout=10, + timeout=HOLMES_SLACKBOT_TIMEOUT_SECONDS, ) response.raise_for_status() - is_connected = bool(response.json().get("integrations")) - _holmes_slackbot_cache['status'] = is_connected - return is_connected + _holmes_slackbot_cache['status'] = bool(response.json().get("integrations")) except Exception as e: logging.warning(f"Failed to get holmes slackbot info {e}") - _holmes_slackbot_cache['status'] = False - return False From 9fade75279aebac351effce9627551ce85243a0a Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 07:12:18 +0000 Subject: [PATCH 4/4] Revert "Cap the Holmes slackbot check at 10s wall-clock" This reverts commit 5e2ddf72d84579c744247445e6f0d6ec9cca48a3. --- src/robusta/core/sinks/robusta/robusta_sink.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/robusta/core/sinks/robusta/robusta_sink.py b/src/robusta/core/sinks/robusta/robusta_sink.py index d0f83e550..668491102 100644 --- a/src/robusta/core/sinks/robusta/robusta_sink.py +++ b/src/robusta/core/sinks/robusta/robusta_sink.py @@ -51,7 +51,6 @@ RUNNER_GET_HOLMES_SLACKBOT_INFO = f"{ROBUSTA_API_ENDPOINT}/api/holmes/integrations/slack/runner" HOLMES_SLACKBOT_CACHE_TTL = int(os.getenv("HOLMES_SLACKBOT_CACHE_TTL", 15 * 60)) -HOLMES_SLACKBOT_TIMEOUT_SECONDS = 10 # Define the cache with a single slot and the configured TTL _holmes_slackbot_cache = TTLCache(maxsize=1, ttl=HOLMES_SLACKBOT_CACHE_TTL) @@ -816,14 +815,6 @@ def __update_job(self, new_job: Job, operation: K8sOperationType): def is_holmes_slackbot_connected(self) -> bool: if 'status' in _holmes_slackbot_cache: return _holmes_slackbot_cache['status'] - # Called under the Slack grouping lock; never block alerts. - _holmes_slackbot_cache['status'] = False - status_check = threading.Thread(target=self.__update_holmes_slackbot_status, daemon=True) - status_check.start() - status_check.join(timeout=HOLMES_SLACKBOT_TIMEOUT_SECONDS) - return _holmes_slackbot_cache.get('status', False) - - def __update_holmes_slackbot_status(self): try: message_json = { "session_token": self.dal.get_session_token(), @@ -833,9 +824,13 @@ def __update_holmes_slackbot_status(self): RUNNER_GET_HOLMES_SLACKBOT_INFO, json=message_json, headers={"Content-Type": "application/json"}, - timeout=HOLMES_SLACKBOT_TIMEOUT_SECONDS, + timeout=10, ) response.raise_for_status() - _holmes_slackbot_cache['status'] = bool(response.json().get("integrations")) + is_connected = bool(response.json().get("integrations")) + _holmes_slackbot_cache['status'] = is_connected + return is_connected except Exception as e: logging.warning(f"Failed to get holmes slackbot info {e}") + _holmes_slackbot_cache['status'] = False + return False