feat(client): add connect_timeout and retry on httpx.ConnectTimeout

changelog/+connect-timeout-retry.fixed.md

@@ -0,0 +1 @@
+Treat `httpx.ConnectTimeout` like other connection failures: it is raised as `ServerNotReachableError` and retried when `retry_on_failure` is enabled, instead of escaping as a raw httpx exception.

changelog/+connect-timeout.added.md

@@ -0,0 +1 @@
+Add the `connect_timeout` setting (`INFRAHUB_CONNECT_TIMEOUT`, default 10 seconds) so the TCP/TLS connection phase of every request is bounded separately from the request `timeout`. An unreachable address now fails fast instead of consuming the whole request budget, which lets endpoints published through DNS round robin fall over to the next address quickly.

docs/docs/python-sdk/reference/config.mdx

@@ -216,12 +216,21 @@ The following settings can be defined in the `Config` class
 ## timeout
 
 <!-- vale on -->
-**Description**: Default connection timeout in seconds<br />
+**Description**: Default request timeout in seconds, applied to the read, write and pool phases of a request.<br />
 **Type**: `integer`<br />
 **Default value**: 60<br />
 **Environment variable**: `INFRAHUB_TIMEOUT`<br />
 <!-- vale off -->
 
+## connect_timeout
+
+<!-- vale on -->
+**Description**: Timeout in seconds for establishing the TCP/TLS connection to Infrahub. Kept short so an unreachable address fails fast; it never exceeds the request timeout.<br />
+**Type**: `integer`<br />
+**Default value**: 10<br />
+**Environment variable**: `INFRAHUB_CONNECT_TIMEOUT`<br />
+<!-- vale off -->
+
 ## transport
 
 <!-- vale on -->

infrahub_sdk/client.py

@@ -273,6 +273,16 @@ def _merge_request_headers(self, headers: dict | None) -> dict:
             merged.update(headers)
         return merged
 
+    def _build_timeout(self, timeout: int) -> httpx.Timeout:
+        """Build the httpx timeout for a single request.
+
+        ``timeout`` bounds the read, write and pool phases. The connect phase (TCP and TLS
+        handshake) is bounded by ``config.connect_timeout`` instead, so an unreachable address
+        fails fast rather than consuming the whole request budget, and the caller's ``timeout``
+        remains an upper bound for it.
+        """
+        return httpx.Timeout(timeout, connect=min(self.config.connect_timeout, timeout))
+
     @property
     def request_context(self) -> RequestContext | None:
         return self._request_context
@@ -1492,8 +1502,10 @@ async def send() -> httpx.Response:
             _rewind_multipart_files(files)
             async with httpx.AsyncClient(**self._build_proxy_config(), verify=self.config.tls_context) as client:
                 try:
-                    return await client.post(url=url, headers=headers, timeout=timeout, files=files)
-                except httpx.NetworkError as exc:
+                    return await client.post(
+                        url=url, headers=headers, timeout=self._build_timeout(timeout), files=files
+                    )
+                except (httpx.NetworkError, httpx.ConnectTimeout) as exc:
                     raise ServerNotReachableError(address=self.address) from exc
                 except httpx.ReadTimeout as exc:
                     raise ServerNotResponsiveError(url=url, timeout=timeout) from exc
@@ -1580,7 +1592,7 @@ async def send() -> httpx.Response:
                 # the caller and closed afterwards.
                 stack = AsyncExitStack()
                 response = await stack.enter_async_context(
-                    client.stream(method="GET", url=url, headers=headers, timeout=request_timeout)
+                    client.stream(method="GET", url=url, headers=headers, timeout=self._build_timeout(request_timeout))
                 )
                 if response.status_code == 429:
                     try:
@@ -1599,7 +1611,7 @@ async def send() -> httpx.Response:
                     stack = open_stream.get("stack")
                     if stack is not None:
                         await stack.aclose()
-            except httpx.NetworkError as exc:
+            except (httpx.NetworkError, httpx.ConnectTimeout) as exc:
                 raise ServerNotReachableError(address=self.address) from exc
             except httpx.ReadTimeout as exc:
                 raise ServerNotResponsiveError(url=url, timeout=request_timeout) from exc
@@ -1637,10 +1649,10 @@ async def _default_request_method(
                     method=method.value,
                     url=url,
                     headers=headers,
-                    timeout=timeout,
+                    timeout=self._build_timeout(timeout),
                     **params,
                 )
-            except httpx.NetworkError as exc:
+            except (httpx.NetworkError, httpx.ConnectTimeout) as exc:
                 raise ServerNotReachableError(address=self.address) from exc
             except httpx.ReadTimeout as exc:
                 raise ServerNotResponsiveError(url=url, timeout=timeout) from exc
@@ -2478,8 +2490,8 @@ def send() -> httpx.Response:
             _rewind_multipart_files(files)
             with httpx.Client(**self._build_proxy_config(), verify=self.config.tls_context) as client:
                 try:
-                    return client.post(url=url, headers=headers, timeout=timeout, files=files)
-                except httpx.NetworkError as exc:
+                    return client.post(url=url, headers=headers, timeout=self._build_timeout(timeout), files=files)
+                except (httpx.NetworkError, httpx.ConnectTimeout) as exc:
                     raise ServerNotReachableError(address=self.address) from exc
                 except httpx.ReadTimeout as exc:
                     raise ServerNotResponsiveError(url=url, timeout=timeout) from exc
@@ -3722,7 +3734,7 @@ def send() -> httpx.Response:
                 # the caller and closed afterwards.
                 stack = ExitStack()
                 response = stack.enter_context(
-                    client.stream(method="GET", url=url, headers=headers, timeout=request_timeout)
+                    client.stream(method="GET", url=url, headers=headers, timeout=self._build_timeout(request_timeout))
                 )
                 if response.status_code == 429:
                     try:
@@ -3741,7 +3753,7 @@ def send() -> httpx.Response:
                     stack = open_stream.get("stack")
                     if stack is not None:
                         stack.close()
-            except httpx.NetworkError as exc:
+            except (httpx.NetworkError, httpx.ConnectTimeout) as exc:
                 raise ServerNotReachableError(address=self.address) from exc
             except httpx.ReadTimeout as exc:
                 raise ServerNotResponsiveError(url=url, timeout=request_timeout) from exc
@@ -3806,10 +3818,10 @@ def _default_request_method(
                     method=method.value,
                     url=url,
                     headers=headers,
-                    timeout=timeout,
+                    timeout=self._build_timeout(timeout),
                     **params,
                 )
-            except httpx.NetworkError as exc:
+            except (httpx.NetworkError, httpx.ConnectTimeout) as exc:
                 raise ServerNotReachableError(address=self.address) from exc
             except httpx.ReadTimeout as exc:
                 raise ServerNotResponsiveError(url=url, timeout=timeout) from exc

infrahub_sdk/config.py

@@ -90,7 +90,18 @@ class ConfigBase(BaseSettings):
     schema_converge_timeout: int = Field(
         default=60, description="Number of seconds to wait for schema to have converged"
     )
-    timeout: int = Field(default=60, description="Default connection timeout in seconds")
+    timeout: int = Field(
+        default=60,
+        description="Default request timeout in seconds, applied to the read, write and pool phases of a request.",
+    )
+    connect_timeout: int = Field(
+        default=10,
+        gt=0,
+        description=(
+            "Timeout in seconds for establishing the TCP/TLS connection to Infrahub. Kept short so an "
+            "unreachable address fails fast; it never exceeds the request timeout."
+        ),
+    )
     transport: RequesterTransport = Field(
         default=RequesterTransport.HTTPX, description="Set an alternate transport using a predefined option"
     )

tests/unit/sdk/test_connect_timeout.py

@@ -0,0 +1,168 @@
+"""Connect-phase timeout handling on the async and sync clients.
+
+Covers the split connect/request timeout handed to httpx, the cap of the connect timeout by the
+per-request timeout, and the conversion of ``httpx.ConnectTimeout`` into ``ServerNotReachableError``
+on every request path so that ``retry_on_failure`` retries it like any other connection failure.
+"""
+
+from __future__ import annotations
+
+import io
+from typing import TYPE_CHECKING
+
+import httpx
+import pytest
+from pydantic import ValidationError
+
+from infrahub_sdk import Config, InfrahubClient, InfrahubClientSync
+from infrahub_sdk.exceptions import ServerNotReachableError
+from infrahub_sdk.types import HTTPMethod
+
+if TYPE_CHECKING:
+    from pytest_httpx import HTTPXMock
+
+CLIENT_TYPES = ["standard", "sync"]
+
+# ``_request_multipart`` and ``_get_streaming`` build their own httpx client, so each path is driven
+# separately to prove they all share the same timeout and error handling.
+REQUEST_PATHS = ["regular", "multipart", "streaming"]
+
+URL = "http://mock/graphql/main"
+
+
+def _make_client(
+    client_type: str, *, connect_timeout: int = 10, retry_on_failure: bool = False, retry_delay: int = 5
+) -> InfrahubClient | InfrahubClientSync:
+    """Build a client with no ``requester`` override so the real (mocked) httpx transport is used."""
+    cfg = Config(
+        address="http://mock",
+        connect_timeout=connect_timeout,
+        retry_on_failure=retry_on_failure,
+        retry_delay=retry_delay,
+    )
+    if client_type == "standard":
+        return InfrahubClient(config=cfg)
+    return InfrahubClientSync(config=cfg)
+
+
+async def _drive_path(client: InfrahubClient | InfrahubClientSync, path: str, timeout: int) -> httpx.Response:
+    """Send one request on the selected path and return the (fully read) response."""
+    if path == "regular":
+        if isinstance(client, InfrahubClient):
+            return await client._request(url=URL, method=HTTPMethod.POST, headers={}, timeout=timeout, payload={})
+        return client._request(url=URL, method=HTTPMethod.POST, headers={}, timeout=timeout, payload={})
+
+    if path == "multipart":
+        files = {"file": ("upload.bin", io.BytesIO(b"file body"), "application/octet-stream")}
+        if isinstance(client, InfrahubClient):
+            return await client._request_multipart(url=URL, headers={}, timeout=timeout, files=files)
+        return client._request_multipart(url=URL, headers={}, timeout=timeout, files=files)
+
+    if isinstance(client, InfrahubClient):
+        async with client._get_streaming(url=URL, timeout=timeout) as response:
+            await response.aread()
+            return response
+    with client._get_streaming(url=URL, timeout=timeout) as response:
+        response.read()
+        return response
+
+
+# --- Configuration ------------------------------------------------------------------------------
+
+
+def test_default_connect_timeout_is_shorter_than_request_timeout() -> None:
+    config = Config(address="http://mock")
+
+    assert config.connect_timeout == 10
+    assert config.connect_timeout < config.timeout
+
+
+def test_connect_timeout_from_env_var(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("INFRAHUB_CONNECT_TIMEOUT", "3")
+
+    assert Config(address="http://mock").connect_timeout == 3
+
+
+def test_connect_timeout_must_be_positive() -> None:
+    with pytest.raises(ValidationError):
+        Config(address="http://mock", connect_timeout=0)
+
+
+# --- Timeout handed to httpx -------------------------------------------------------------------
+
+
+@pytest.mark.parametrize("path", REQUEST_PATHS)
+@pytest.mark.parametrize("client_type", CLIENT_TYPES)
+async def test_request_uses_connect_timeout_for_connect_phase_only(
+    client_type: str, path: str, httpx_mock: HTTPXMock
+) -> None:
+    """Every request path bounds the connect phase with ``connect_timeout`` and the rest with ``timeout``."""
+    httpx_mock.add_response(status_code=200, json={"data": {}})
+    client = _make_client(client_type, connect_timeout=4)
+
+    await _drive_path(client=client, path=path, timeout=30)
+
+    request = httpx_mock.get_request()
+    assert request is not None
+    assert request.extensions["timeout"] == {"connect": 4.0, "read": 30.0, "write": 30.0, "pool": 30.0}
+
+
+@pytest.mark.parametrize("client_type", CLIENT_TYPES)
+async def test_connect_timeout_is_capped_by_request_timeout(client_type: str, httpx_mock: HTTPXMock) -> None:
+    """A per-request ``timeout`` shorter than ``connect_timeout`` also bounds the connect phase."""
+    httpx_mock.add_response(status_code=200, json={"data": {}})
+    client = _make_client(client_type, connect_timeout=10)
+
+    await _drive_path(client=client, path="regular", timeout=3)
+
+    request = httpx_mock.get_request()
+    assert request is not None
+    assert request.extensions["timeout"]["connect"] == pytest.approx(3.0)
+
+
+# --- ConnectTimeout is a connection failure -----------------------------------------------------
+
+
+@pytest.mark.parametrize("path", REQUEST_PATHS)
+@pytest.mark.parametrize("client_type", CLIENT_TYPES)
+async def test_connect_timeout_raises_server_not_reachable(client_type: str, path: str, httpx_mock: HTTPXMock) -> None:
+    """``httpx.ConnectTimeout`` surfaces as ``ServerNotReachableError`` on every request path."""
+    httpx_mock.add_exception(httpx.ConnectTimeout("timed out"))
+    client = _make_client(client_type)
+
+    with pytest.raises(ServerNotReachableError) as excinfo:
+        await _drive_path(client=client, path=path, timeout=10)
+
+    assert isinstance(excinfo.value.__cause__, httpx.ConnectTimeout)
+    assert excinfo.value.address == "http://mock"
+
+
+@pytest.mark.parametrize("client_type", CLIENT_TYPES)
+async def test_execute_graphql_retries_after_connect_timeout(client_type: str, httpx_mock: HTTPXMock) -> None:
+    """With ``retry_on_failure`` enabled, a connect timeout is retried and the next attempt succeeds."""
+    httpx_mock.add_exception(httpx.ConnectTimeout("timed out"))
+    httpx_mock.add_response(status_code=200, json={"data": {"ok": True}})
+    client = _make_client(client_type, retry_on_failure=True, retry_delay=0)
+
+    if isinstance(client, InfrahubClient):
+        data = await client.execute_graphql(query="query { ok }")
+    else:
+        data = client.execute_graphql(query="query { ok }")
+
+    assert data == {"ok": True}
+    assert len(httpx_mock.get_requests()) == 2
+
+
+@pytest.mark.parametrize("client_type", CLIENT_TYPES)
+async def test_execute_graphql_surfaces_connect_timeout_without_retry(client_type: str, httpx_mock: HTTPXMock) -> None:
+    """Without ``retry_on_failure``, a connect timeout is raised once as ``ServerNotReachableError``."""
+    httpx_mock.add_exception(httpx.ConnectTimeout("timed out"))
+    client = _make_client(client_type)
+
+    with pytest.raises(ServerNotReachableError):
+        if isinstance(client, InfrahubClient):
+            await client.execute_graphql(query="query { ok }")
+        else:
+            client.execute_graphql(query="query { ok }")
+
+    assert len(httpx_mock.get_requests()) == 1