Skip to content

Commit 9cea4de

Browse files
committed
Allow controlling the failover mechanism
1 parent 24b3ed7 commit 9cea4de

18 files changed

Lines changed: 833 additions & 74 deletions

.bumpversion.cfg

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,26 @@
11
[bumpversion]
2-
current_version = 3.1.2
2+
current_version = 3.2.0
33

44
[bumpversion:file:pyproject.toml]
55
search = version = "{current_version}"
66
replace = version = "{new_version}"
77

88
[bumpversion:file:dbutils/__init__.py]
9-
search = __version__ = '{current_version}'
10-
replace = __version__ = '{new_version}'
9+
search = __version__ = "{current_version}"
10+
replace = __version__ = "{new_version}"
1111

1212
[bumpversion:file:README.md]
1313
search = The current version {current_version}
1414
replace = The current version {new_version}
1515

1616
[bumpversion:file:docs/main.rst]
1717
search = :Version: {current_version}
18-
search = :Version: {new_version}
18+
replace = :Version: {new_version}
1919

2020
[bumpversion:file:docs/main.de.rst]
2121
search = :Version: {current_version}
22-
search = :Version: {new_version}
22+
replace = :Version: {new_version}
23+
24+
[bumpversion:file:tox.ini]
25+
search = dbutils-{current_version}-py3-none-any.whl
26+
replace = dbutils-{new_version}-py3-none-any.whl

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ to a database that can be used in all kinds of multi-threaded environments.
77
The suite supports DB-API 2 compliant database interfaces
88
and the classic PyGreSQL interface.
99

10-
The current version 3.1.2 of DBUtils supports Python versions 3.7 to 3.14.
10+
The current version 3.2.0 of DBUtils supports Python versions 3.7 to 3.14.
1111

1212
**Please have a look at the [changelog](https://webwareforpython.github.io/DBUtils/changelog.html), because there were some breaking changes in version 2.0.**
1313

dbutils/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@
22

33
__all__ = ["__version__"]
44

5-
__version__ = "3.1.2"
5+
__version__ = "3.2.0"

dbutils/persistent_db.py

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@
4444
for which the connection failover mechanism shall be applied,
4545
if the default (OperationalError, InterfaceError, InternalError)
4646
is not adequate for the used database module
47+
isfatal: an optional callable deciding whether a given exception is
48+
really fatal for the connection and the failover mechanism shall
49+
be applied (see the steady_db module for details)
4750
ping: an optional flag controlling when connections are checked
4851
with the ping() method if such a method is available
4952
(0 = None = never, 1 = default = whenever it is requested,
@@ -146,7 +149,8 @@ class PersistentDB:
146149
def __init__(
147150
self, creator,
148151
maxusage=None, setsession=None, failures=None, ping=1,
149-
closeable=False, threadlocal=None, *args, **kwargs):
152+
closeable=False, threadlocal=None, isfatal=None,
153+
*args, **kwargs):
150154
"""Set up the persistent DB-API 2 connection generator.
151155
152156
creator: either an arbitrary function returning new DB-API 2
@@ -160,6 +164,15 @@ def __init__(
160164
for which the connection failover mechanism shall be applied,
161165
if the default (OperationalError, InterfaceError, InternalError)
162166
is not adequate for the used database module
167+
isfatal: an optional callable that is passed an exception matching
168+
the failures and shall return whether that error is really fatal
169+
for the connection, i.e. whether the failover mechanism shall be
170+
applied. Use this to distinguish errors that merely report a
171+
failed statement (such as a deliberate server side statement
172+
timeout) from errors that report a broken connection, when the
173+
database module maps both onto the same exception class. The
174+
callable must tolerate arbitrary exception instances, including
175+
instances without args.
163176
ping: determines when the connection should be checked with ping()
164177
(0 = None = never, 1 = default = whenever it is requested,
165178
2 = when a cursor is created, 4 = when a query is executed,
@@ -191,6 +204,7 @@ def __init__(
191204
self._maxusage = maxusage
192205
self._setsession = setsession
193206
self._failures = failures
207+
self._isfatal = isfatal
194208
self._ping = ping
195209
self._closeable = closeable
196210
self._args, self._kwargs = args, kwargs
@@ -201,7 +215,7 @@ def steady_connection(self):
201215
return connect(
202216
self._creator, self._maxusage, self._setsession,
203217
self._failures, self._ping, self._closeable,
204-
*self._args, **self._kwargs)
218+
self._isfatal, *self._args, **self._kwargs)
205219

206220
def connection(self, shareable=False): # noqa: ARG002
207221
"""Get a steady, persistent DB-API 2 connection.

dbutils/pooled_db.py

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@
5959
for which the connection failover mechanism shall be applied,
6060
if the default (OperationalError, InterfaceError, InternalError)
6161
is not adequate for the used database module
62+
isfatal: an optional callable deciding whether a given exception is
63+
really fatal for the connection and the failover mechanism shall
64+
be applied (see the steady_db module for details)
6265
ping: an optional flag controlling when connections are checked
6366
with the ping() method if such a method is available
6467
(0 = None = never, 1 = default = whenever fetched from the pool,
@@ -195,7 +198,7 @@ def __init__(
195198
self, creator, mincached=0, maxcached=0,
196199
maxshared=0, maxconnections=0, blocking=False,
197200
maxusage=None, setsession=None, reset=True,
198-
failures=None, ping=1,
201+
failures=None, ping=1, isfatal=None,
199202
*args, **kwargs):
200203
"""Set up the DB-API 2 connection pool.
201204
@@ -227,6 +230,15 @@ def __init__(
227230
for which the connection failover mechanism shall be applied,
228231
if the default (OperationalError, InterfaceError, InternalError)
229232
is not adequate for the used database module
233+
isfatal: an optional callable that is passed an exception matching
234+
the failures and shall return whether that error is really fatal
235+
for the connection, i.e. whether the failover mechanism shall be
236+
applied. Use this to distinguish errors that merely report a
237+
failed statement (such as a deliberate server side statement
238+
timeout) from errors that report a broken connection, when the
239+
database module maps both onto the same exception class. The
240+
callable must tolerate arbitrary exception instances, including
241+
instances without args.
230242
ping: determines when the connection should be checked with ping()
231243
(0 = None = never, 1 = default = whenever fetched from the pool,
232244
2 = when a cursor is created, 4 = when a query is executed,
@@ -256,6 +268,7 @@ def __init__(
256268
self._setsession = setsession
257269
self._reset = reset
258270
self._failures = failures
271+
self._isfatal = isfatal
259272
self._ping = ping
260273
if mincached is None:
261274
mincached = 0
@@ -291,7 +304,8 @@ def steady_connection(self):
291304
"""Get a steady, unpooled DB-API 2 connection."""
292305
return connect(
293306
self._creator, self._maxusage, self._setsession,
294-
self._failures, self._ping, True, *self._args, **self._kwargs)
307+
self._failures, self._ping, True, self._isfatal,
308+
*self._args, **self._kwargs)
295309

296310
def connection(self, shareable=True):
297311
"""Get a steady, cached DB-API 2 connection from the pool.

dbutils/steady_db.py

Lines changed: 84 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@
9090
"""
9191

9292
import sys
93-
from contextlib import suppress
93+
from contextlib import contextmanager, suppress
9494

9595
from . import __version__
9696

@@ -109,7 +109,8 @@ class InvalidCursorError(SteadyDBError):
109109

110110
def connect(
111111
creator, maxusage=None, setsession=None,
112-
failures=None, ping=1, closeable=True, *args, **kwargs):
112+
failures=None, ping=1, closeable=True, isfatal=None,
113+
*args, **kwargs):
113114
"""Create a "tough" connection.
114115
115116
A hardened version of the connection function of a DB-API 2 module.
@@ -126,6 +127,17 @@ def connect(
126127
for which the failover mechanism shall be applied, if the default
127128
(OperationalError, InternalError, Interface) is not adequate
128129
for the used database module
130+
isfatal: an optional callable that is passed an exception matching the
131+
failures and shall return whether that error is really fatal for
132+
the connection, i.e. whether the failover mechanism shall be
133+
applied. Since the failures can only be specified as exception
134+
classes, and Python cannot exclude subclasses from an except
135+
clause, this is the only way to distinguish errors that merely
136+
report a failed statement (such as a deliberate server side
137+
statement timeout) from errors that report a broken connection,
138+
when the database module maps both onto the same exception class.
139+
The callable must tolerate arbitrary exception instances of the
140+
classes given as failures, including instances without args.
129141
ping: determines when the connection should be checked with ping()
130142
(0 = None = never, 1 = default = when _ping_check() is called,
131143
2 = whenever a cursor is created, 4 = when a query is executed,
@@ -137,7 +149,7 @@ def connect(
137149
"""
138150
return SteadyDBConnection(
139151
creator, maxusage, setsession,
140-
failures, ping, closeable, *args, **kwargs)
152+
failures, ping, closeable, isfatal, *args, **kwargs)
141153

142154

143155
class SteadyDBConnection:
@@ -147,7 +159,8 @@ class SteadyDBConnection:
147159

148160
def __init__(
149161
self, creator, maxusage=None, setsession=None,
150-
failures=None, ping=1, closeable=True, *args, **kwargs):
162+
failures=None, ping=1, closeable=True, isfatal=None,
163+
*args, **kwargs):
151164
"""Create a "tough" DB-API 2 connection."""
152165
# basic initialization to make finalizer work
153166
self._con = None
@@ -191,6 +204,9 @@ def __init__(
191204
failures, tuple) and not issubclass(failures, Exception):
192205
raise TypeError("'failures' must be a tuple of exceptions.")
193206
self._failures = failures
207+
if isfatal is not None and not callable(isfatal):
208+
raise TypeError("'isfatal' must be a callable.")
209+
self._isfatal = isfatal
194210
self._ping = ping if isinstance(ping, int) else 0
195211
self._closeable = closeable
196212
self._args, self._kwargs = args, kwargs
@@ -376,6 +392,20 @@ def dbapi(self):
376392
" (please set creator.dbapi).")
377393
return self._dbapi
378394

395+
@property
396+
def dbapi_connection(self):
397+
"""Return the underlying DB-API 2 connection.
398+
399+
Note that operations executed directly on this connection are
400+
not covered by the failover mechanism and are not counted
401+
towards the maximum usage limit of the steady connection.
402+
"""
403+
return self._con
404+
405+
def _fatal_check(self, error):
406+
"""Check whether the error is fatal for the connection."""
407+
return self._isfatal is None or self._isfatal(error)
408+
379409
def threadsafety(self):
380410
"""Return the thread safety level of the connection."""
381411
if self._threadsafety is None:
@@ -475,13 +505,19 @@ def _cursor(self, *args, **kwargs):
475505
transaction = self._transaction
476506
if not transaction:
477507
self._ping_check(2)
508+
overused = False
478509
try:
479510
# check whether the connection has been used too often
480511
if (self._maxusage and self._usage >= self._maxusage
481512
and not transaction):
513+
overused = True
482514
raise self._failure
483515
cursor = self._con.cursor(*args, **kwargs) # try to get a cursor
484516
except self._failures as error: # error in getting cursor
517+
# the connection must always be reset when it is overused,
518+
# otherwise the application can veto the failover mechanism
519+
if not overused and not self._fatal_check(error):
520+
raise
485521
try: # try to reopen the connection
486522
con = self._create()
487523
except Exception: # noqa: S110
@@ -525,6 +561,8 @@ def __init__(self, con, *args, **kwargs):
525561
# basic initialization to make finalizer work
526562
self._cursor = None
527563
self._closed = True
564+
# nesting level of no_failover() contexts
565+
self._no_failover = 0
528566
# proper initialization of the cursor
529567
self._con = con
530568
self._args, self._kwargs = args, kwargs
@@ -551,6 +589,41 @@ def __iter__(self):
551589
except TypeError: # create iterator if not provided
552590
return iter(cursor.fetchone, None)
553591

592+
@property
593+
def dbapi_cursor(self):
594+
"""Return the underlying DB-API 2 cursor.
595+
596+
Note that operations executed directly on this cursor are not
597+
covered by the failover mechanism and are not counted towards
598+
the maximum usage limit of the underlying steady connection.
599+
Consider using no_failover() instead, which keeps the bookkeeping
600+
of the steady connection intact.
601+
"""
602+
cursor = self._cursor
603+
if not cursor:
604+
raise InvalidCursorError
605+
return cursor
606+
607+
@contextmanager
608+
def no_failover(self):
609+
"""Suspend the failover mechanism inside this context.
610+
611+
Statements that are executed on this cursor inside the context
612+
are not retried when they fail, but the error is raised to the
613+
application immediately. Everything else, particularly the ping
614+
check and the usage bookkeeping of the underlying steady
615+
connection, is left untouched.
616+
617+
This is useful for statements that are expected to fail in a
618+
controlled way, e.g. when a deliberate server side timeout has
619+
been requested, where a retry would multiply the runtime.
620+
"""
621+
self._no_failover += 1
622+
try:
623+
yield self
624+
finally:
625+
self._no_failover -= 1
626+
554627
def setinputsizes(self, sizes):
555628
"""Store input sizes in case cursor needs to be reopened."""
556629
self._inputsizes = sizes
@@ -594,10 +667,12 @@ def tough_method(*args, **kwargs):
594667
transaction = con._transaction
595668
if not transaction:
596669
con._ping_check(4)
670+
overused = False
597671
try:
598672
# check whether the connection has been used too often
599673
if (con._maxusage and con._usage >= con._maxusage
600674
and not transaction):
675+
overused = True
601676
raise con._failure
602677
if execute:
603678
self._setsizes()
@@ -606,6 +681,11 @@ def tough_method(*args, **kwargs):
606681
if execute:
607682
self._clearsizes()
608683
except con._failures as error: # execution error
684+
# the connection must always be reset when it is overused,
685+
# otherwise the application can veto the failover mechanism
686+
if not overused and (
687+
self._no_failover or not con._fatal_check(error)):
688+
raise
609689
if not transaction:
610690
try:
611691
cursor2 = con._cursor(

0 commit comments

Comments
 (0)