Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions ckanext/csvwmapandtransform/db.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@ def init(db_uri: str = "", echo=False):
if not db_uri:
db_uri = toolkit.config.get("ckanext.csvwmapandtransform.db_url")
global ENGINE, _METADATA, JOBS_TABLE, METADATA_TABLE, LOGS_TABLE
ENGINE = sqlalchemy.create_engine(db_uri, echo=echo, convert_unicode=True)
_METADATA = sqlalchemy.MetaData(ENGINE)
ENGINE = sqlalchemy.create_engine(db_uri, echo=echo)
_METADATA = sqlalchemy.MetaData()
JOBS_TABLE = _init_jobs_table()
METADATA_TABLE = _init_metadata_table()
LOGS_TABLE = _init_logs_table()
Expand Down Expand Up @@ -109,10 +109,10 @@ def get_job(job_id):
result_dict = {
field: (
value.isoformat()
if isinstance(value := getattr(result, field), datetime.datetime)
if isinstance(value := result._mapping[field], datetime.datetime)
else value
)
for field in result.keys()
for field in result._mapping.keys()
}

result_dict["metadata"] = _get_metadata(job_id)
Expand Down Expand Up @@ -231,7 +231,7 @@ def _update_job(job_id, job_dict):
if "data" in job_dict:
job_dict["data"] = six.text_type(job_dict["data"])

with ENGINE.connect() as conn:
with ENGINE.begin() as conn:
conn.execute(
JOBS_TABLE.update().where(JOBS_TABLE.c.job_id == job_id).values(**job_dict)
)
Expand Down Expand Up @@ -375,7 +375,7 @@ def _get_metadata(job_id):

metadata = {
row["key"]: json.loads(row["value"]) if row["type"] == "json" else row["value"]
for row in results
for row in (r._mapping for r in results)
}
return metadata

Expand All @@ -389,7 +389,7 @@ def _get_logs(job_id):
LOGS_TABLE.select().where(LOGS_TABLE.c.job_id == job_id)
).fetchall()

results = [dict(result) for result in results]
results = [dict(result._mapping) for result in results]

for result in results:
result.pop("job_id")
Expand Down
25 changes: 12 additions & 13 deletions ckanext/csvwmapandtransform/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,6 @@ def __init__(self, task_id, input):
self.input = input

def emit(self, record):
conn = db.ENGINE.connect()
try:
# Turn strings into unicode to stop SQLAlchemy
# "Unicode type received non-unicode bind param value" warnings.
Expand All @@ -329,21 +328,21 @@ def emit(self, record):
module = str(record.module)
funcName = str(record.funcName)

conn.execute(
db.LOGS_TABLE.insert().values(
job_id=self.task_id,
timestamp=datetime.datetime.utcnow(),
message=message,
level=level,
module=module,
funcName=funcName,
lineno=record.lineno,
# begin() commits on success; SQLAlchemy 2 no longer autocommits.
with db.ENGINE.begin() as conn:
conn.execute(
db.LOGS_TABLE.insert().values(
job_id=self.task_id,
timestamp=datetime.datetime.utcnow(),
message=message,
level=level,
module=module,
funcName=funcName,
lineno=record.lineno,
)
)
)
except Exception as e:
sys.stderr.write("StoringHandler DB write failed: {}\n".format(e))
finally:
conn.close()


class DatetimeJsonEncoder(json.JSONEncoder):
Expand Down
Loading