From 22cd8605ec34e9f3bd1514e70590766eb3926e29 Mon Sep 17 00:00:00 2001 From: Asim Awdah Date: Sun, 25 Oct 2026 17:43:48 +0300 Subject: [PATCH 1/4] Add jitter to database retry backoff Spread sync and async database retries across clients during upstream error bursts. Add deterministic tests for jittered backoff and caps. --- src/replit/database/database.py | 24 ++++++++++++++++++++++-- tests/test_database_retry.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 tests/test_database_retry.py diff --git a/src/replit/database/database.py b/src/replit/database/database.py index 83ffa62..30f13da 100644 --- a/src/replit/database/database.py +++ b/src/replit/database/database.py @@ -3,6 +3,7 @@ import asyncio from collections import abc import json +import random import threading from typing import ( Any, @@ -23,6 +24,25 @@ from urllib3.filepost import encode_multipart_formdata +class _JitteredExponentialRetry(ExponentialRetry): + """Add jitter to async exponential backoff to spread retry traffic.""" + + def get_timeout(self, attempt: int, response: Any = None) -> float: + """Return an exponential delay with up to 100% random jitter.""" + timeout = super().get_timeout(attempt, response) + return min(timeout + random.uniform(0, timeout), self._max_timeout) + + +class _JitteredRetry(Retry): + """Add jitter to urllib3 retry backoff to spread retry traffic.""" + + def get_backoff_time(self) -> float: + """Return the usual backoff plus up to 100% random jitter.""" + timeout = super().get_backoff_time() + jitter_range = max(timeout, self.backoff_factor) + return min(timeout + random.uniform(0, jitter_range), self.backoff_max) + + def to_primitive(o: Any) -> Any: """If object is an observed object, converts to primitve, otherwise returns it. @@ -106,7 +126,7 @@ def __init__( self._get_db_url = get_db_url self._unbind = unbind - retry_options = ExponentialRetry(attempts=retry_count) + retry_options = _JitteredExponentialRetry(attempts=retry_count) self.client = RetryClient(client_session=self.sess, retry_options=retry_options) if self._get_db_url: @@ -532,7 +552,7 @@ def __init__( self.sess = requests.Session() self._get_db_url = get_db_url self._unbind = unbind - retries = Retry( + retries = _JitteredRetry( total=retry_count, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504] ) self.sess.mount("http://", HTTPAdapter(max_retries=retries)) diff --git a/tests/test_database_retry.py b/tests/test_database_retry.py new file mode 100644 index 0000000..39e6c0a --- /dev/null +++ b/tests/test_database_retry.py @@ -0,0 +1,31 @@ +"""Tests for database retry backoff.""" + +from unittest import TestCase +from unittest.mock import patch + +from replit.database.database import _JitteredExponentialRetry, _JitteredRetry + + +class TestJitteredExponentialRetry(TestCase): + """Tests for async database retry backoff.""" + + def test_adds_jitter_to_exponential_timeout(self) -> None: + """Async retry delays include jitter while respecting their cap.""" + retry = _JitteredExponentialRetry( + attempts=3, start_timeout=1, factor=2, max_timeout=3 + ) + + with patch("replit.database.database.random.uniform", return_value=1.5): + self.assertEqual(retry.get_timeout(attempt=1), 3) + + +class TestJitteredRetry(TestCase): + """Tests for synchronous database retry backoff.""" + + def test_adds_jitter_to_first_retry(self) -> None: + """The first synchronous retry is delayed by random jitter.""" + retry = _JitteredRetry(total=3, backoff_factor=0.1) + retry = retry.increment(method="GET", url="/") + + with patch("replit.database.database.random.uniform", return_value=0.05): + self.assertEqual(retry.get_backoff_time(), 0.05) From c0693f3285eae63412908567fbfdd09122f69183 Mon Sep 17 00:00:00 2001 From: Asim Awdah Date: Sun, 25 Oct 2026 18:03:03 +0300 Subject: [PATCH 2/4] Suppress false positive on retry jitter Bandit flags pseudo-random timing jitter as security-sensitive. Document that retry jitter only spreads request timing and suppress B311 at the two call sites. --- src/replit/database/database.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/replit/database/database.py b/src/replit/database/database.py index 30f13da..272325d 100644 --- a/src/replit/database/database.py +++ b/src/replit/database/database.py @@ -30,7 +30,9 @@ class _JitteredExponentialRetry(ExponentialRetry): def get_timeout(self, attempt: int, response: Any = None) -> float: """Return an exponential delay with up to 100% random jitter.""" timeout = super().get_timeout(attempt, response) - return min(timeout + random.uniform(0, timeout), self._max_timeout) + # Retry timing jitter does not protect secrets. + jitter = random.uniform(0, timeout) # nosec B311 + return min(timeout + jitter, self._max_timeout) class _JitteredRetry(Retry): @@ -40,7 +42,9 @@ def get_backoff_time(self) -> float: """Return the usual backoff plus up to 100% random jitter.""" timeout = super().get_backoff_time() jitter_range = max(timeout, self.backoff_factor) - return min(timeout + random.uniform(0, jitter_range), self.backoff_max) + # Retry timing jitter does not protect secrets. + jitter = random.uniform(0, jitter_range) # nosec B311 + return min(timeout + jitter, self.backoff_max) def to_primitive(o: Any) -> Any: From a9f2244fa97e1d08477ce0cf8ef4c95d3edfa3ba Mon Sep 17 00:00:00 2001 From: Asim Awdah Date: Sun, 25 Oct 2026 18:06:53 +0300 Subject: [PATCH 3/4] Use secure randomness for retry jitter Use SystemRandom for retry jitter so Bandit does not flag the timing distribution as cryptographically weak. Keep tests deterministic by patching the generator. --- src/replit/database/database.py | 10 +++++----- tests/test_database_retry.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/replit/database/database.py b/src/replit/database/database.py index 272325d..dfdbc18 100644 --- a/src/replit/database/database.py +++ b/src/replit/database/database.py @@ -3,7 +3,7 @@ import asyncio from collections import abc import json -import random +import secrets import threading from typing import ( Any, @@ -23,6 +23,8 @@ from requests.adapters import HTTPAdapter, Retry from urllib3.filepost import encode_multipart_formdata +_SYSTEM_RANDOM = secrets.SystemRandom() + class _JitteredExponentialRetry(ExponentialRetry): """Add jitter to async exponential backoff to spread retry traffic.""" @@ -30,8 +32,7 @@ class _JitteredExponentialRetry(ExponentialRetry): def get_timeout(self, attempt: int, response: Any = None) -> float: """Return an exponential delay with up to 100% random jitter.""" timeout = super().get_timeout(attempt, response) - # Retry timing jitter does not protect secrets. - jitter = random.uniform(0, timeout) # nosec B311 + jitter = _SYSTEM_RANDOM.uniform(0, timeout) return min(timeout + jitter, self._max_timeout) @@ -42,8 +43,7 @@ def get_backoff_time(self) -> float: """Return the usual backoff plus up to 100% random jitter.""" timeout = super().get_backoff_time() jitter_range = max(timeout, self.backoff_factor) - # Retry timing jitter does not protect secrets. - jitter = random.uniform(0, jitter_range) # nosec B311 + jitter = _SYSTEM_RANDOM.uniform(0, jitter_range) return min(timeout + jitter, self.backoff_max) diff --git a/tests/test_database_retry.py b/tests/test_database_retry.py index 39e6c0a..e02c4c7 100644 --- a/tests/test_database_retry.py +++ b/tests/test_database_retry.py @@ -15,7 +15,7 @@ def test_adds_jitter_to_exponential_timeout(self) -> None: attempts=3, start_timeout=1, factor=2, max_timeout=3 ) - with patch("replit.database.database.random.uniform", return_value=1.5): + with patch("replit.database.database._SYSTEM_RANDOM.uniform", return_value=1.5): self.assertEqual(retry.get_timeout(attempt=1), 3) @@ -27,5 +27,5 @@ def test_adds_jitter_to_first_retry(self) -> None: retry = _JitteredRetry(total=3, backoff_factor=0.1) retry = retry.increment(method="GET", url="/") - with patch("replit.database.database.random.uniform", return_value=0.05): + with patch("replit.database.database._SYSTEM_RANDOM.uniform", return_value=0.05): self.assertEqual(retry.get_backoff_time(), 0.05) From ed92bbc7ef691621835e87aa2c8287e6d6de407a Mon Sep 17 00:00:00 2001 From: Asim Awdah Date: Sun, 25 Oct 2026 18:20:43 +0300 Subject: [PATCH 4/4] Format retry tests with Black --- tests/test_database_retry.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_database_retry.py b/tests/test_database_retry.py index e02c4c7..367c63e 100644 --- a/tests/test_database_retry.py +++ b/tests/test_database_retry.py @@ -27,5 +27,7 @@ def test_adds_jitter_to_first_retry(self) -> None: retry = _JitteredRetry(total=3, backoff_factor=0.1) retry = retry.increment(method="GET", url="/") - with patch("replit.database.database._SYSTEM_RANDOM.uniform", return_value=0.05): + with patch( + "replit.database.database._SYSTEM_RANDOM.uniform", return_value=0.05 + ): self.assertEqual(retry.get_backoff_time(), 0.05)