From 801eddbde7190ff02b12bff46090f26f2f2a007e Mon Sep 17 00:00:00 2001 From: Thomas Hanke Date: Mon, 16 Nov 2026 14:38:42 +0100 Subject: [PATCH] fix: port job database code to SQLAlchemy 2.0 CKAN 2.12 ships SQLAlchemy 2.x, which removed APIs this module still used (the same code was copied into ckanext-fuseki, where queuing a job failed with "Invalid argument(s) 'convert_unicode' sent to create_engine()"). - create_engine: drop convert_unicode. - MetaData(): no longer bound to an engine. - _update_job and the log handler wrote on a plain connection, which is no longer committed automatically; use engine.begin(). - Rows no longer support string indexing or dict(row); use row._mapping. - Log handler: only swallow Exception instead of a bare except. Verified with SQLAlchemy 2.0.51 against SQLite: add job, metadata, log records, complete/error/failed-to-post and lookup. --- ckanext/csvwmapandtransform/db.py | 14 +++++++------- ckanext/csvwmapandtransform/tasks.py | 27 +++++++++++++-------------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/ckanext/csvwmapandtransform/db.py b/ckanext/csvwmapandtransform/db.py index 1bfe0b7..ff08679 100644 --- a/ckanext/csvwmapandtransform/db.py +++ b/ckanext/csvwmapandtransform/db.py @@ -46,8 +46,8 @@ def init(db_uri: str = "", echo=False): if not db_uri: db_uri = toolkit.config.get("ckanext.csvwmapandtransform.db_url") global ENGINE, _METADATA, JOBS_TABLE, METADATA_TABLE, LOGS_TABLE - ENGINE = sqlalchemy.create_engine(db_uri, echo=echo, convert_unicode=True) - _METADATA = sqlalchemy.MetaData(ENGINE) + ENGINE = sqlalchemy.create_engine(db_uri, echo=echo) + _METADATA = sqlalchemy.MetaData() JOBS_TABLE = _init_jobs_table() METADATA_TABLE = _init_metadata_table() LOGS_TABLE = _init_logs_table() @@ -109,10 +109,10 @@ def get_job(job_id): result_dict = { field: ( value.isoformat() - if isinstance(value := getattr(result, field), datetime.datetime) + if isinstance(value := result._mapping[field], datetime.datetime) else value ) - for field in result.keys() + for field in result._mapping.keys() } result_dict["metadata"] = _get_metadata(job_id) @@ -231,7 +231,7 @@ def _update_job(job_id, job_dict): if "data" in job_dict: job_dict["data"] = six.text_type(job_dict["data"]) - with ENGINE.connect() as conn: + with ENGINE.begin() as conn: conn.execute( JOBS_TABLE.update().where(JOBS_TABLE.c.job_id == job_id).values(**job_dict) ) @@ -375,7 +375,7 @@ def _get_metadata(job_id): metadata = { row["key"]: json.loads(row["value"]) if row["type"] == "json" else row["value"] - for row in results + for row in (r._mapping for r in results) } return metadata @@ -389,7 +389,7 @@ def _get_logs(job_id): LOGS_TABLE.select().where(LOGS_TABLE.c.job_id == job_id) ).fetchall() - results = [dict(result) for result in results] + results = [dict(result._mapping) for result in results] for result in results: result.pop("job_id") diff --git a/ckanext/csvwmapandtransform/tasks.py b/ckanext/csvwmapandtransform/tasks.py index 72bbe5e..4048ef1 100644 --- a/ckanext/csvwmapandtransform/tasks.py +++ b/ckanext/csvwmapandtransform/tasks.py @@ -223,7 +223,6 @@ def __init__(self, task_id, input): self.input = input def emit(self, record): - conn = db.ENGINE.connect() try: # Turn strings into unicode to stop SQLAlchemy # "Unicode type received non-unicode bind param value" warnings. @@ -232,21 +231,21 @@ def emit(self, record): module = str(record.module) funcName = str(record.funcName) - conn.execute( - db.LOGS_TABLE.insert().values( - job_id=self.task_id, - timestamp=datetime.datetime.utcnow(), - message=message, - level=level, - module=module, - funcName=funcName, - lineno=record.lineno, + # begin() commits on success; SQLAlchemy 2 no longer autocommits. + with db.ENGINE.begin() as conn: + conn.execute( + db.LOGS_TABLE.insert().values( + job_id=self.task_id, + timestamp=datetime.datetime.utcnow(), + message=message, + level=level, + module=module, + funcName=funcName, + lineno=record.lineno, + ) ) - ) - except: + except Exception: pass - finally: - conn.close() class DatetimeJsonEncoder(json.JSONEncoder):