From fcc2f5c2142beca19fc8cb90652551b31d9a4b42 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Mon, 8 Jun 2026 23:20:10 +0800 Subject: [PATCH] Fix probe timeout alert handling --- main.py | 36 +++++++++++++++++++++++ notifications/telegram.py | 2 ++ tests/test_request_handling.py | 52 ++++++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+) diff --git a/main.py b/main.py index 44a4e98..be18b6f 100644 --- a/main.py +++ b/main.py @@ -618,6 +618,10 @@ def connect_ib(): return build_broker_adapters().connect_ib() +def _build_health_probe_connection_error_message(exc: Exception) -> str: + return f"{t('health_probe_title')}\n{t('ibkr_connection_error_prefix')}{str(exc)}" + + def log_runtime_event(log_context, event, **fields): return build_composer().build_reporting_adapters().log_event(log_context, event, **fields) @@ -1158,6 +1162,38 @@ def _handle_probe(*, response_body: str = "Probe OK"): positions_count=len(positions), ) return response_body, 200 + except (ConnectionError, TimeoutError) as exc: + if report is not None: + append_runtime_report_error( + report, + stage="health_probe", + message=str(exc), + error_type=type(exc).__name__, + failure_category="ibkr_connection", + ) + finalize_runtime_report( + report, + status="error", + diagnostics={"probe_failure_category": "ibkr_connection"}, + ) + if log_context is not None: + log_runtime_event( + log_context, + "health_probe_failed", + message="Health probe IBKR connection failed", + severity="ERROR", + execution_window="probe", + error_type=type(exc).__name__, + error_message=str(exc), + failure_category="ibkr_connection", + ) + error_msg = _build_health_probe_connection_error_message(exc) + _publish_runtime_failure_notification( + detailed_text=error_msg, + compact_text=error_msg, + exc=exc, + ) + return "Error", 500 except Exception as exc: if report is not None: append_runtime_report_error( diff --git a/notifications/telegram.py b/notifications/telegram.py index 24ec785..a43b609 100644 --- a/notifications/telegram.py +++ b/notifications/telegram.py @@ -10,6 +10,7 @@ "error_title": "🚨 【策略异常】", "health_probe_title": "🔎 【连接探针】", "health_probe_error_prefix": "健康探针异常:\n", + "ibkr_connection_error_prefix": "IBKR 连接异常:\n", "canary_title": "🐤 【金丝雀检查】", "strategy_label": "🧭 策略: {name}", "account_ids_detail": "🆔 账户: {account_ids}", @@ -151,6 +152,7 @@ "error_title": "🚨 【Strategy Error】", "health_probe_title": "🔎 【Health Probe】", "health_probe_error_prefix": "Health probe error:\n", + "ibkr_connection_error_prefix": "IBKR connection error:\n", "canary_title": "🐤 【Canary Check】", "strategy_label": "🧭 Strategy: {name}", "account_ids_detail": "🆔 Account: {account_ids}", diff --git a/tests/test_request_handling.py b/tests/test_request_handling.py index 5f8767b..3b90ace 100644 --- a/tests/test_request_handling.py +++ b/tests/test_request_handling.py @@ -210,6 +210,58 @@ def disconnect(self): assert observed["notifications"] == [] +def test_handle_probe_connect_timeout_sends_concise_connection_notification(strategy_module, monkeypatch): + observed = {"events": [], "notifications": []} + timeout_message = ( + "IBKR API handshake timed out after TCP preflight succeeded " + "for 172.16.159.2:4001 clientId=231" + ) + + monkeypatch.setattr(strategy_module, "build_request_log_context", lambda: types.SimpleNamespace(run_id="run-001")) + monkeypatch.setattr(strategy_module, "build_execution_report", lambda log_context, **_kwargs: {"status": "pending"}) + monkeypatch.setattr( + strategy_module, + "persist_execution_report", + lambda report, **_kwargs: observed.setdefault("report", dict(report)) or "/tmp/runtime-report.json", + ) + monkeypatch.setattr( + strategy_module, + "log_runtime_event", + lambda context, event, **fields: observed["events"].append((event, fields)), + ) + monkeypatch.setattr(strategy_module, "load_strategy_plugin_signals", lambda: ((), None)) + monkeypatch.setattr(strategy_module, "attach_strategy_plugin_report", lambda *args, **kwargs: None) + monkeypatch.setattr( + strategy_module, + "connect_ib", + lambda: (_ for _ in ()).throw(TimeoutError(timeout_message)), + ) + monkeypatch.setattr( + strategy_module, + "publish_notification", + lambda **kwargs: observed["notifications"].append(kwargs), + ) + + with strategy_module.app.test_request_context("/probe", method="POST"): + body, status = strategy_module.handle_probe() + + assert status == 500 + assert body == "Error" + assert observed["report"]["status"] == "error" + assert observed["report"]["errors"][0]["stage"] == "health_probe" + assert observed["report"]["errors"][0]["failure_category"] == "ibkr_connection" + assert [event for event, _fields in observed["events"]] == [ + "health_probe_received", + "health_probe_failed", + ] + assert observed["events"][1][1]["failure_category"] == "ibkr_connection" + assert len(observed["notifications"]) == 1 + detailed_text = observed["notifications"][0]["detailed_text"] + assert "IBKR" in detailed_text + assert timeout_message in detailed_text + assert "Traceback" not in detailed_text + + def test_handle_probe_failure_sends_notification(strategy_module, monkeypatch): observed = {"events": [], "notifications": []}