From 040415b14a23e6749f985f6f3599941e3ddb3165 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Mon, 4 May 2026 16:19:06 +0200 Subject: [PATCH 01/41] first part of the provenance collection draft --- src/hermes/commands/harvest/base.py | 67 ++++++++++- src/hermes/model/provenance/ld_prov.py | 158 +++++++++++++++++++++++++ src/hermes/model/types/ld_list.py | 2 +- 3 files changed, 225 insertions(+), 2 deletions(-) create mode 100644 src/hermes/model/provenance/ld_prov.py diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index 0d3d9e5f..93b23601 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -12,6 +12,8 @@ from hermes.error import HermesPluginRunError, MisconfigurationError from hermes.model.context_manager import HermesContext from hermes.model import SoftwareMetadata +from hermes.model.provenance.ld_prov import ld_prov_list, ld_prov_node +from hermes.model.types.ld_context import ALL_CONTEXTS class HermesHarvestPlugin(HermesPlugin): @@ -19,10 +21,19 @@ class HermesHarvestPlugin(HermesPlugin): TODO: describe the harvesting process and how this is mapped to this plugin. """ + def __init__(self): + self.io_operations: list[tuple[dict, dict, dict]] = [] + super().__init__() def __call__(self, command: HermesCommand) -> SoftwareMetadata: pass + def load(): + pass + + def write(): + pass + class HarvestSettings(BaseModel): """Generic harvesting settings.""" @@ -37,9 +48,11 @@ class HermesHarvestCommand(HermesCommand): settings_class = HarvestSettings def __call__(self, args: argparse.Namespace) -> None: - self.log.info("# Metadata harvesting") self.args = args + self.log.info("# Load provenance from old harvest or create new document.") + prov_doc, base_plugin = self.init_provenance_document() + self.log.info("# Metadata harvesting") if len(self.settings.sources) == 0: self.log.critical("# No harvest plugin was configured to be run and loaded.") raise MisconfigurationError("No harvest plugin was configured to be run and loaded.") @@ -66,6 +79,24 @@ def __call__(self, args: argparse.Namespace) -> None: except Exception: self.log.exception(f"### Unknown error while executing the {plugin_name} plugin, skipping it now.") continue + self.remove_provenance_info_for_plugin(prov_doc, plugin_name) + + plugin = prov_doc.add_hermes_plugin("harvest", plugin_name) + plugin_io_operations = plugin_func.io_operations # liste von drei Tupeln (input_file, load_function, output) + for plugin_io_operation in plugin_io_operations: + loaded_source = prov_doc.add_entity() + loaded_source.update(plugin_io_operation[0]) + io_op = prov_doc.add_activity() + plugin_io_operation[1]["prov:wasAssociatedWith"] = [base_plugin.ref, plugin.ref] + plugin_io_operation[1]["prov:used"] = loaded_source.ref + io_op.update(plugin_io_operation[1]) + loaded_data = prov_doc.add_entity() + plugin_io_operation[2].update({ + "prov:wasAttributedTo": plugin.ref, + "prov:wasDerivedFrom": loaded_source.ref, + "prov:wasGeneratedBy": io_op.ref + }) + loaded_data.update(plugin_io_operations[2]) self.log.info(f"### Store metadata harvested by {plugin_name} plugin") # store harvested data @@ -76,3 +107,37 @@ def __call__(self, args: argparse.Namespace) -> None: if not harvested_any: self.log.critical("No harvest plugin ran successfully.") raise HermesPluginRunError("No harvest plugin ran successfully.") + + def init_provenance_document(self) -> tuple[ld_prov_list, ld_prov_node]: + ctx = HermesContext() + ctx.prepare_step("harvest") + with ctx["provenance"] as cache: + try: + ld_prov_doc = ld_prov_list.from_list(cache["codemeta"], container_type="@graph", context=ALL_CONTEXTS) + return ld_prov_doc, ld_prov_doc.shallow_search({"schema:name": lambda doc, node: node["schema:name"][0].find("harvest base plugin") != -1})[0] + except KeyError: + pass + prov_doc = ld_prov_list() + prov_doc.init_hermes_agents() + return prov_doc, prov_doc.add_hermes_base_plugin("harvest") + + def remove_provenance_info_for_plugin(self, prov_doc, plugin) -> None: + plugin = prov_doc.shallow_search({ + "schema:name": (lambda doc, node: f"harvest plugin {plugin}" in node["schema:name"]), + }) + if len(plugin) == 0: + return + # two passes are needed because the nodes are nested exactly two levels + related = prov_doc.shallow_search({ + "prov:wasAssociatedWith": (lambda doc, node: plugin.ref in node["prov:wasAssociatedWith"]), + "prov:wasAttributedTo": (lambda doc, node: plugin.ref in node["prov:wasAttributedTo"]) + }) + ids = [plugin.ref, *(rel.ref for rel in related)] + related = prov_doc.shallow_search({ + f"prov:{key}": (lambda doc, node: any(id in node[f"prov:{key}"] for id in ids)) for key in [ + "wasAssociatedWith", "wasAttributedTo", "wasGeneratedBy", "used", "wasDerivedFrom", "wasInformedBy" + ] + }) + for item in related: + items = prov_doc.shallow_search({"@id": (lambda doc, node: node["@id"] == item["@id"])}) + del prov_doc[items[0].index] diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py new file mode 100644 index 00000000..2c2a10f7 --- /dev/null +++ b/src/hermes/model/provenance/ld_prov.py @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: 2026 German Aerospace Center (DLR) +# +# SPDX-License-Identifier: Apache-2.0 + +# SPDX-FileContributor: Michael Fritzsche + +from typing import Optional, Union +from typing_extensions import Self +import uuid + +from hermes import utils +from hermes.model.types import ld_dict, ld_list +from hermes.model.types.ld_container import BASIC_TYPE, EXPANDED_JSON_LD_VALUE, JSON_LD_CONTEXT_DICT, TIME_TYPE +from hermes.model.types.ld_context import ALL_CONTEXTS, iri_map + + +class ld_prov_container: + def _to_python( + self: Self, + full_iri: str, + ld_value: Union[EXPANDED_JSON_LD_VALUE, dict[str, EXPANDED_JSON_LD_VALUE], list[str], str] + ) -> Union["ld_prov_node", "ld_prov_list", BASIC_TYPE, TIME_TYPE]: + item = super()._to_python(full_iri, ld_value) + if isinstance(item, ld_list): + return ld_prov_list( + data=item._data, parent=item.parent, key=item.key, index=item.index, context=item.context + ) + elif isinstance(item, ld_dict): + return ld_prov_node( + data=item._data, parent=item.parent, key=item.key, index=item.index, context=item.context + ) + return item + + +class ld_prov_list(ld_list): + NODE_IRI_FORMAT = "graph://{uuid}/{index}" + PROV_DOC_IRI = iri_map['hermes-rt', "graph"] + + def __init__( + self: Self, + *, + data: EXPANDED_JSON_LD_VALUE = [{"@graph": []}], + parent: Optional[Union[ld_dict, ld_list]] = None, + key: Optional[str] = PROV_DOC_IRI, + index: Optional[int] = None, + context: Optional[list[Union[str, JSON_LD_CONTEXT_DICT]]] = ALL_CONTEXTS + ) -> None: + self.id = uuid.uuid1() + self.node_index = 0 + super().__init__([{"@graph": []}], parent=parent, key=key, index=index, context=context) + + def __getitem__( + self: Self, index: Union[int, slice] + ) -> Union[ + BASIC_TYPE, + TIME_TYPE, + "ld_prov_node", + "ld_prov_list", + list[Union[BASIC_TYPE, TIME_TYPE, "ld_prov_node", "ld_prov_list"]] + ]: + item = super().__getitem__(index) + if isinstance(item, ld_list): + return ld_prov_list( + data=item._data, parent=item.parent, key=item.key, index=item.index, context=item.context + ) + elif isinstance(item, ld_dict): + return ld_prov_node( + data=item._data, parent=item.parent, key=item.key, index=item.index, context=item.context + ) + return item + + def next_node_iri(self) -> str: + self.node_index += 1 + return self.NODE_IRI_FORMAT.format(uuid=self.id, index=self.node_index) + + def add_activity(self) -> "ld_prov_node": + self.append({"@id": self.next_node_iri(), "@type": "prov:Activity"}) + return self[-1] + + def add_agent(self) -> "ld_prov_node": + self.append({"@id": self.next_node_iri(), "@type": "prov:Agent"}) + return self[-1] + + def add_entity(self) -> "ld_prov_node": + self.append({"@id": self.next_node_iri(), "@type": "prov:Entity"}) + return self[-1] + + def init_hermes_agents(self) -> "ld_prov_node": + hermes = self.add_agent() + hermes.update({ + "schema:name": utils.hermes_name, + "schema:version": utils.hermes_version, + "schema:url": utils.hermes_urls, + }) + hermes["@type"].append("schema:SoftwareApplication") + node = self.add_agent() + node.update({ + "schema:name": utils.hermes_name + " cache", + "schema:version": utils.hermes_version, + "prov:actedOnBehalfOf": hermes.ref + }) + node["@type"].append("schema:SoftwareApplication") + return node + + def add_hermes_command(self, step) -> "ld_prov_node": + node = self.add_agent() + node.update({ + "schema:name": f"{utils.hermes_name} {step} command", + "schema:version": utils.hermes_version, + "prov:actedOnBehalfOf": self.shallow_search( + {"schema:name": (lambda doc, node: node["schema:name"] == utils.hermes_name)} + ) + }) + node["@type"].append("schema:SoftwareApplication") + return node + + def add_hermes_base_plugin(self, step) -> "ld_prov_node": + node = self.add_agent() + node.update({ + "schema:name": f"{utils.hermes_name} {step} base plugin", + "schema:version": utils.hermes_version, + "prov:actedOnBehalfOf": self.shallow_search( + {"schema:name": (lambda doc, node: node["schema:name"] == f"{utils.hermes_name} {step} command")} + ) + }) + node["@type"].append("schema:SoftwareApplication") + return node + + def add_hermes_plugin(self, step, name) -> "ld_prov_node": + node = self.add_agent() + # TODO: add version + node.update({ + "schema:name": f"{utils.hermes_name} {step} plugin '{name}'", + "prov:actedOnBehalfOf": self.shallow_search( + {"schema:name": (lambda doc, node: node["schema:name"] == f"{utils.hermes_name} {step} base plugin")} + ) + }) + node["@type"].append("schema:SoftwareApplication") + return node + + def shallow_search(self, query: dict) -> list["ld_prov_node"]: + return [ + item for item in self for key, test in query.items() if key in item and test(self, item) + ] + + +class ld_prov_node(ld_dict): + def __init__( + self: Self, + data: list[dict[str, EXPANDED_JSON_LD_VALUE]], + *, + parent: Optional[Union[ld_dict, ld_list]] = None, + key: Optional[str] = None, + index: Optional[int] = None, + context: Optional[list[Union[str, JSON_LD_CONTEXT_DICT]]] = ALL_CONTEXTS + ) -> None: + self.id = uuid.uuid1() + super().__init__(data, parent=parent, key=key, index=index, context=context) diff --git a/src/hermes/model/types/ld_list.py b/src/hermes/model/types/ld_list.py index 14331472..003cda82 100644 --- a/src/hermes/model/types/ld_list.py +++ b/src/hermes/model/types/ld_list.py @@ -589,7 +589,7 @@ def from_list( key: Optional[str] = None, context: Optional[Union[str, JSON_LD_CONTEXT_DICT, list[Union[str, JSON_LD_CONTEXT_DICT]]]] = None, container_type: str = "@set" - ) -> ld_list: + ) -> Self: """ Creates a ld_list from the given list with the given parent, key, context and container_type.\n Note that only container_type '@set' is valid for key '@type'.\n From a96821eb7964dfa9c1802c3049cae1e48a3d70b1 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Mon, 4 May 2026 17:21:28 +0200 Subject: [PATCH 02/41] first draft for provenance recording of harvest command --- src/hermes/commands/harvest/base.py | 64 +++++++++++++++++++++++--- src/hermes/model/provenance/ld_prov.py | 12 ++--- 2 files changed, 64 insertions(+), 12 deletions(-) diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index 93b23601..61e2b3e7 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -79,10 +79,18 @@ def __call__(self, args: argparse.Namespace) -> None: except Exception: self.log.exception(f"### Unknown error while executing the {plugin_name} plugin, skipping it now.") continue + + self.log.info(f"### Store metadata harvested by {plugin_name} plugin") + # store harvested data + harvested_data.write_to_cache(ctx, plugin_name) + harvested_any = True + self.remove_provenance_info_for_plugin(prov_doc, plugin_name) plugin = prov_doc.add_hermes_plugin("harvest", plugin_name) - plugin_io_operations = plugin_func.io_operations # liste von drei Tupeln (input_file, load_function, output) + plugin_io_operations = plugin_func.io_operations + outputs = [] + io_ops = [] for plugin_io_operation in plugin_io_operations: loaded_source = prov_doc.add_entity() loaded_source.update(plugin_io_operation[0]) @@ -97,11 +105,52 @@ def __call__(self, args: argparse.Namespace) -> None: "prov:wasGeneratedBy": io_op.ref }) loaded_data.update(plugin_io_operations[2]) + outputs.append(loaded_data.ref) + io_ops.append(io_op.ref) + + map_activity = prov_doc.add_activity() + map_activity.update({ + "prov:wasInformedBy": io_ops, + "prov:used": outputs, + "prov:wasAssociatedWith": plugin.ref + }) + data_output = prov_doc.add_entity() + data_output.update({ + "prov:wasAttributedTo": plugin.ref, + "prov:wasGeneratedBy": map_activity.ref, + "prov:wasDerivedFrom": outputs + }) + + write = prov_doc.add_activity() + write.update({ + "prov:wasAssociatedWith": [ + prov_doc.shallow_search({ + "schema:name": lambda doc, node: node["schema:name"][0].find("harvest command") != -1 + })[0].ref, + prov_doc.shallow_search({ + "schema:name": lambda doc, node: node["schema:name"][0].find(" cache") != -1 + })[0].ref, + plugin.ref + ], + "prov:used": data_output.ref, + "prov:wasInformedBy": map_activity.ref + }) + # TODO: add more info + write_output = prov_doc.add_entity() + write_output.update({ + "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref + }) + write_output = prov_doc.add_entity() + write_output.update({ + "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref + }) + write_output = prov_doc.add_entity() + write_output.update({ + "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref + }) - self.log.info(f"### Store metadata harvested by {plugin_name} plugin") - # store harvested data - harvested_data.write_to_cache(ctx, plugin_name) - harvested_any = True + with ctx["provenance"] as cache: + cache["codemeta"] = prov_doc.ld_value ctx.finalize_step('harvest') if not harvested_any: @@ -114,11 +163,14 @@ def init_provenance_document(self) -> tuple[ld_prov_list, ld_prov_node]: with ctx["provenance"] as cache: try: ld_prov_doc = ld_prov_list.from_list(cache["codemeta"], container_type="@graph", context=ALL_CONTEXTS) - return ld_prov_doc, ld_prov_doc.shallow_search({"schema:name": lambda doc, node: node["schema:name"][0].find("harvest base plugin") != -1})[0] + return ld_prov_doc, ld_prov_doc.shallow_search({ + "schema:name": lambda doc, node: node["schema:name"][0].find("harvest base plugin") != -1 + })[0] except KeyError: pass prov_doc = ld_prov_list() prov_doc.init_hermes_agents() + prov_doc.add_hermes_command("harvest") return prov_doc, prov_doc.add_hermes_base_plugin("harvest") def remove_provenance_info_for_plugin(self, prov_doc, plugin) -> None: diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index 2c2a10f7..a8419da9 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -108,8 +108,8 @@ def add_hermes_command(self, step) -> "ld_prov_node": "schema:name": f"{utils.hermes_name} {step} command", "schema:version": utils.hermes_version, "prov:actedOnBehalfOf": self.shallow_search( - {"schema:name": (lambda doc, node: node["schema:name"] == utils.hermes_name)} - ) + {"schema:name": (lambda doc, node: node["schema:name"][0] == utils.hermes_name)} + )[0].ref }) node["@type"].append("schema:SoftwareApplication") return node @@ -120,8 +120,8 @@ def add_hermes_base_plugin(self, step) -> "ld_prov_node": "schema:name": f"{utils.hermes_name} {step} base plugin", "schema:version": utils.hermes_version, "prov:actedOnBehalfOf": self.shallow_search( - {"schema:name": (lambda doc, node: node["schema:name"] == f"{utils.hermes_name} {step} command")} - ) + {"schema:name": (lambda doc, node: node["schema:name"][0] == f"{utils.hermes_name} {step} command")} + )[0].ref }) node["@type"].append("schema:SoftwareApplication") return node @@ -132,8 +132,8 @@ def add_hermes_plugin(self, step, name) -> "ld_prov_node": node.update({ "schema:name": f"{utils.hermes_name} {step} plugin '{name}'", "prov:actedOnBehalfOf": self.shallow_search( - {"schema:name": (lambda doc, node: node["schema:name"] == f"{utils.hermes_name} {step} base plugin")} - ) + {"schema:name": (lambda doc, node: node["schema:name"][0] == f"{utils.hermes_name} {step} base plugin")} + )[0].ref }) node["@type"].append("schema:SoftwareApplication") return node From 4f85996d53dda0997706283c3a6aa6682b889999 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Thu, 7 May 2026 16:33:13 +0200 Subject: [PATCH 03/41] improved ld_prov_list and adjusted the collection of provenance accordingly --- src/hermes/commands/harvest/base.py | 83 ++++------ src/hermes/model/provenance/ld_prov.py | 216 ++++++++++++------------- 2 files changed, 135 insertions(+), 164 deletions(-) diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index 61e2b3e7..e6fbfc4c 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -12,7 +12,7 @@ from hermes.error import HermesPluginRunError, MisconfigurationError from hermes.model.context_manager import HermesContext from hermes.model import SoftwareMetadata -from hermes.model.provenance.ld_prov import ld_prov_list, ld_prov_node +from hermes.model.provenance.ld_prov import ld_prov_list from hermes.model.types.ld_context import ALL_CONTEXTS @@ -50,7 +50,8 @@ class HermesHarvestCommand(HermesCommand): def __call__(self, args: argparse.Namespace) -> None: self.args = args self.log.info("# Load provenance from old harvest or create new document.") - prov_doc, base_plugin = self.init_provenance_document() + prov_doc = self.init_provenance_document() + base_plugin = prov_doc.get_hermes_base_plugin("harvest") self.log.info("# Metadata harvesting") if len(self.settings.sources) == 0: @@ -92,60 +93,48 @@ def __call__(self, args: argparse.Namespace) -> None: outputs = [] io_ops = [] for plugin_io_operation in plugin_io_operations: - loaded_source = prov_doc.add_entity() - loaded_source.update(plugin_io_operation[0]) - io_op = prov_doc.add_activity() - plugin_io_operation[1]["prov:wasAssociatedWith"] = [base_plugin.ref, plugin.ref] - plugin_io_operation[1]["prov:used"] = loaded_source.ref - io_op.update(plugin_io_operation[1]) - loaded_data = prov_doc.add_entity() + loaded_source = prov_doc.add_entity(data=plugin_io_operation[0]) + plugin_io_operation[1].update( + {"prov:wasAssociatedWith": [base_plugin.ref, plugin.ref], "prov:used": loaded_source.ref} + ) + io_op = prov_doc.add_activity(data=plugin_io_operation[1]) plugin_io_operation[2].update({ "prov:wasAttributedTo": plugin.ref, "prov:wasDerivedFrom": loaded_source.ref, "prov:wasGeneratedBy": io_op.ref }) - loaded_data.update(plugin_io_operations[2]) + loaded_data = prov_doc.add_entity(data=plugin_io_operations[2]) outputs.append(loaded_data.ref) io_ops.append(io_op.ref) - map_activity = prov_doc.add_activity() - map_activity.update({ + map_activity = prov_doc.add_activity(data={ "prov:wasInformedBy": io_ops, "prov:used": outputs, "prov:wasAssociatedWith": plugin.ref }) - data_output = prov_doc.add_entity() - data_output.update({ + data_output = prov_doc.add_entity(data={ "prov:wasAttributedTo": plugin.ref, "prov:wasGeneratedBy": map_activity.ref, "prov:wasDerivedFrom": outputs }) - write = prov_doc.add_activity() - write.update({ + write = prov_doc.add_activity(data={ "prov:wasAssociatedWith": [ - prov_doc.shallow_search({ - "schema:name": lambda doc, node: node["schema:name"][0].find("harvest command") != -1 - })[0].ref, - prov_doc.shallow_search({ - "schema:name": lambda doc, node: node["schema:name"][0].find(" cache") != -1 - })[0].ref, + prov_doc.get_hermes_command("harvest").ref, + prov_doc.get_hermes_cache().ref, plugin.ref ], "prov:used": data_output.ref, "prov:wasInformedBy": map_activity.ref }) # TODO: add more info - write_output = prov_doc.add_entity() - write_output.update({ + prov_doc.add_entity(data={ "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref }) - write_output = prov_doc.add_entity() - write_output.update({ + prov_doc.add_entity(data={ "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref }) - write_output = prov_doc.add_entity() - write_output.update({ + prov_doc.add_entity(data={ "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref }) @@ -157,39 +146,37 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.critical("No harvest plugin ran successfully.") raise HermesPluginRunError("No harvest plugin ran successfully.") - def init_provenance_document(self) -> tuple[ld_prov_list, ld_prov_node]: + def init_provenance_document(self) -> ld_prov_list: ctx = HermesContext() ctx.prepare_step("harvest") with ctx["provenance"] as cache: try: ld_prov_doc = ld_prov_list.from_list(cache["codemeta"], container_type="@graph", context=ALL_CONTEXTS) - return ld_prov_doc, ld_prov_doc.shallow_search({ - "schema:name": lambda doc, node: node["schema:name"][0].find("harvest base plugin") != -1 - })[0] + return ld_prov_doc except KeyError: pass prov_doc = ld_prov_list() prov_doc.init_hermes_agents() - prov_doc.add_hermes_command("harvest") - return prov_doc, prov_doc.add_hermes_base_plugin("harvest") - - def remove_provenance_info_for_plugin(self, prov_doc, plugin) -> None: - plugin = prov_doc.shallow_search({ - "schema:name": (lambda doc, node: f"harvest plugin {plugin}" in node["schema:name"]), - }) - if len(plugin) == 0: + return prov_doc + + def remove_provenance_info_for_plugin(self, prov_doc: ld_prov_list, plugin) -> None: + plugin = prov_doc.get_hermes_plugin("harvest", plugin) + if plugin is None: return # two passes are needed because the nodes are nested exactly two levels - related = prov_doc.shallow_search({ - "prov:wasAssociatedWith": (lambda doc, node: plugin.ref in node["prov:wasAssociatedWith"]), - "prov:wasAttributedTo": (lambda doc, node: plugin.ref in node["prov:wasAttributedTo"]) - }) + related = prov_doc.shallow_search(lambda doc, node: ( + ("prov:wasAssociatedWith" in node and plugin.ref in node["prov:wasAssociatedWith"]) or + ("prov:wasAttributedTo" in node and plugin.ref in node["prov:wasAttributedTo"]) + )) + if len(related) == 0: + del prov_doc[plugin.index] + return ids = [plugin.ref, *(rel.ref for rel in related)] - related = prov_doc.shallow_search({ - f"prov:{key}": (lambda doc, node: any(id in node[f"prov:{key}"] for id in ids)) for key in [ + related = prov_doc.shallow_search(lambda doc, node: any( + (f"prov:{key}" in node and id in node[f"prov:{key}"]) for id in ids for key in [ "wasAssociatedWith", "wasAttributedTo", "wasGeneratedBy", "used", "wasDerivedFrom", "wasInformedBy" ] - }) + )) for item in related: - items = prov_doc.shallow_search({"@id": (lambda doc, node: node["@id"] == item["@id"])}) + items = prov_doc.shallow_search(lambda doc, node: ("@id" in node and node["@id"] == item["@id"])) del prov_doc[items[0].index] diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index a8419da9..df4b8788 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -6,35 +6,22 @@ from typing import Optional, Union from typing_extensions import Self -import uuid from hermes import utils from hermes.model.types import ld_dict, ld_list -from hermes.model.types.ld_container import BASIC_TYPE, EXPANDED_JSON_LD_VALUE, JSON_LD_CONTEXT_DICT, TIME_TYPE +from hermes.model.types.ld_container import EXPANDED_JSON_LD_VALUE, JSON_LD_CONTEXT_DICT from hermes.model.types.ld_context import ALL_CONTEXTS, iri_map -class ld_prov_container: - def _to_python( - self: Self, - full_iri: str, - ld_value: Union[EXPANDED_JSON_LD_VALUE, dict[str, EXPANDED_JSON_LD_VALUE], list[str], str] - ) -> Union["ld_prov_node", "ld_prov_list", BASIC_TYPE, TIME_TYPE]: - item = super()._to_python(full_iri, ld_value) - if isinstance(item, ld_list): - return ld_prov_list( - data=item._data, parent=item.parent, key=item.key, index=item.index, context=item.context - ) - elif isinstance(item, ld_dict): - return ld_prov_node( - data=item._data, parent=item.parent, key=item.key, index=item.index, context=item.context - ) - return item - - class ld_prov_list(ld_list): - NODE_IRI_FORMAT = "graph://{uuid}/{index}" + NODE_IRI_FORMAT = "_:{type}/{index}" + HERMES_ID = f"https://doi.org/{utils.hermes_doi}" + HERMES_CACHE_ID = "_:hermes/cache" + HERMES_COMMAND_ID_FORMAT = "_:hermes/command/{step}" + HERMES_PLUGIN_ID_FORMAT = "_:hermes/plugin/{step}/{name}" + HERMES_BASE_PLUGIN_ID_FORMAT = "_:hermes/base_plugin/{step}" PROV_DOC_IRI = iri_map['hermes-rt', "graph"] + INDICES = {} def __init__( self: Self, @@ -45,114 +32,111 @@ def __init__( index: Optional[int] = None, context: Optional[list[Union[str, JSON_LD_CONTEXT_DICT]]] = ALL_CONTEXTS ) -> None: - self.id = uuid.uuid1() - self.node_index = 0 super().__init__([{"@graph": []}], parent=parent, key=key, index=index, context=context) - def __getitem__( - self: Self, index: Union[int, slice] - ) -> Union[ - BASIC_TYPE, - TIME_TYPE, - "ld_prov_node", - "ld_prov_list", - list[Union[BASIC_TYPE, TIME_TYPE, "ld_prov_node", "ld_prov_list"]] - ]: - item = super().__getitem__(index) - if isinstance(item, ld_list): - return ld_prov_list( - data=item._data, parent=item.parent, key=item.key, index=item.index, context=item.context - ) - elif isinstance(item, ld_dict): - return ld_prov_node( - data=item._data, parent=item.parent, key=item.key, index=item.index, context=item.context - ) - return item - - def next_node_iri(self) -> str: - self.node_index += 1 - return self.NODE_IRI_FORMAT.format(uuid=self.id, index=self.node_index) - - def add_activity(self) -> "ld_prov_node": - self.append({"@id": self.next_node_iri(), "@type": "prov:Activity"}) - return self[-1] - - def add_agent(self) -> "ld_prov_node": - self.append({"@id": self.next_node_iri(), "@type": "prov:Agent"}) - return self[-1] - - def add_entity(self) -> "ld_prov_node": - self.append({"@id": self.next_node_iri(), "@type": "prov:Entity"}) - return self[-1] - - def init_hermes_agents(self) -> "ld_prov_node": - hermes = self.add_agent() - hermes.update({ + def next_node_iri(self, type) -> str: + if type not in ld_prov_list.INDICES: + ld_prov_list.INDICES[type] = 0 + ld_prov_list.INDICES[type] += 1 + return self.NODE_IRI_FORMAT.format(type=type, index=ld_prov_list.INDICES[type]) + + def add_activity(self, *, data={}) -> ld_dict: + self.append(data) + activity = self[-1] + if "@type" not in data: + activity["@type"] = "prov:Activity" + else: + activity["@type"].append("prov:Activity") + if "@id" not in data: + activity["@id"] = self.next_node_iri("Activity") + return activity + + def add_agent(self, *, data={}) -> ld_dict: + self.append(data) + agent = self[-1] + if "@type" not in data: + agent["@type"] = "prov:Agent" + else: + agent["@type"].append("prov:Agent") + if "@id" not in data: + agent["@id"] = self.next_node_iri("Agent") + return agent + + def add_entity(self, *, data={}) -> ld_dict: + self.append(data) + entity = self[-1] + if "@type" not in data: + entity["@type"] = "prov:Entity" + else: + entity["@type"].append("prov:Entity") + if "@id" not in data: + entity["@id"] = self.next_node_iri("Entity") + return entity + + def init_hermes_agents(self) -> None: + hermes = self.add_agent(data={ + "@id": ld_prov_list.HERMES_ID, + "@type": "schema:SoftwareApplication", "schema:name": utils.hermes_name, "schema:version": utils.hermes_version, - "schema:url": utils.hermes_urls, + "schema:url": [*set(utils.hermes_urls.values())] }) - hermes["@type"].append("schema:SoftwareApplication") - node = self.add_agent() - node.update({ + self.add_agent(data={ + "@id": ld_prov_list.HERMES_CACHE_ID, + "@type": "schema:SoftwareApplication", "schema:name": utils.hermes_name + " cache", "schema:version": utils.hermes_version, "prov:actedOnBehalfOf": hermes.ref }) - node["@type"].append("schema:SoftwareApplication") - return node - - def add_hermes_command(self, step) -> "ld_prov_node": - node = self.add_agent() - node.update({ - "schema:name": f"{utils.hermes_name} {step} command", - "schema:version": utils.hermes_version, - "prov:actedOnBehalfOf": self.shallow_search( - {"schema:name": (lambda doc, node: node["schema:name"][0] == utils.hermes_name)} - )[0].ref - }) - node["@type"].append("schema:SoftwareApplication") - return node - - def add_hermes_base_plugin(self, step) -> "ld_prov_node": - node = self.add_agent() - node.update({ - "schema:name": f"{utils.hermes_name} {step} base plugin", - "schema:version": utils.hermes_version, - "prov:actedOnBehalfOf": self.shallow_search( - {"schema:name": (lambda doc, node: node["schema:name"][0] == f"{utils.hermes_name} {step} command")} - )[0].ref - }) - node["@type"].append("schema:SoftwareApplication") - return node - - def add_hermes_plugin(self, step, name) -> "ld_prov_node": - node = self.add_agent() + for step in ["harvest", "process", "curate", "deposit", "postprocess"]: + command = self.add_agent(data={ + "@id": ld_prov_list.HERMES_COMMAND_ID_FORMAT.format(step=step), + "@type": "schema:SoftwareApplication", + "schema:name": f"{utils.hermes_name} {step} command", + "schema:version": utils.hermes_version, + "prov:actedOnBehalfOf": hermes.ref + }) + self.add_agent(data={ + "@id": ld_prov_list.HERMES_BASE_PLUGIN_ID_FORMAT.format(step=step), + "@type": "schema:SoftwareApplication", + "schema:name": f"{utils.hermes_name} {step} base plugin", + "schema:version": utils.hermes_version, + "prov:actedOnBehalfOf": command.ref + }) + + def add_hermes_plugin(self, step, name) -> ld_dict: # TODO: add version - node.update({ + node = self.add_agent(data={ + "@id": ld_prov_list.HERMES_PLUGIN_ID_FORMAT.format(step=step, name=name), + "@type": "schema:SoftwareApplication", "schema:name": f"{utils.hermes_name} {step} plugin '{name}'", - "prov:actedOnBehalfOf": self.shallow_search( - {"schema:name": (lambda doc, node: node["schema:name"][0] == f"{utils.hermes_name} {step} base plugin")} - )[0].ref + "prov:actedOnBehalfOf": self.get_hermes_base_plugin(step) }) - node["@type"].append("schema:SoftwareApplication") return node - def shallow_search(self, query: dict) -> list["ld_prov_node"]: - return [ - item for item in self for key, test in query.items() if key in item and test(self, item) - ] + def shallow_search(self, query) -> list[ld_dict]: + return [item for item in self if query(self, item)] + def get_hermes(self) -> ld_dict: + return self.shallow_search(lambda doc, node: ("@id" in node and node["@id"] == ld_prov_list.HERMES_ID))[0] -class ld_prov_node(ld_dict): - def __init__( - self: Self, - data: list[dict[str, EXPANDED_JSON_LD_VALUE]], - *, - parent: Optional[Union[ld_dict, ld_list]] = None, - key: Optional[str] = None, - index: Optional[int] = None, - context: Optional[list[Union[str, JSON_LD_CONTEXT_DICT]]] = ALL_CONTEXTS - ) -> None: - self.id = uuid.uuid1() - super().__init__(data, parent=parent, key=key, index=index, context=context) + def get_hermes_cache(self) -> ld_dict: + return self.shallow_search(lambda doc, node: ("@id" in node and node["@id"] == ld_prov_list.HERMES_CACHE_ID))[0] + + def get_hermes_base_plugin(self, step) -> ld_dict: + return self.shallow_search(lambda doc, node: ( + "@id" in node and node["@id"] == ld_prov_list.HERMES_BASE_PLUGIN_ID_FORMAT.format(step=step) + ))[0] + + def get_hermes_plugin(self, step, name) -> Union[ld_dict, None]: + search_result = self.shallow_search(lambda doc, node: ( + "@id" in node and node["@id"] == ld_prov_list.HERMES_PLUGIN_ID_FORMAT.format(step=step, name=name) + )) + if search_result: + return search_result[0] + return None + + def get_hermes_command(self, step) -> ld_dict: + return self.shallow_search(lambda doc, node: ( + "@id" in node and node["@id"] == ld_prov_list.HERMES_COMMAND_ID_FORMAT.format(step=step) + ))[0] From 893096c9bbb21f765636ed2f9d09c434fd0d2842 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Thu, 21 May 2026 11:53:36 +0200 Subject: [PATCH 04/41] updated compaction of ld_lists --- src/hermes/model/types/ld_container.py | 2 +- src/hermes/model/types/ld_list.py | 41 ++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/hermes/model/types/ld_container.py b/src/hermes/model/types/ld_container.py index abc37fa9..10d2a82c 100644 --- a/src/hermes/model/types/ld_container.py +++ b/src/hermes/model/types/ld_container.py @@ -316,7 +316,7 @@ def compact( COMPACTED_JSON_LD_VALUE: The compacted version of selfs JSON-LD representation. """ return self.ld_proc.compact( - self.ld_value, context or self.context, {"documentLoader": bundled_loader, "skipExpand": True} + self.ld_value, context or self.full_context, {"documentLoader": bundled_loader, "skipExpand": True} ) def to_python(self): diff --git a/src/hermes/model/types/ld_list.py b/src/hermes/model/types/ld_list.py index 003cda82..01a1c265 100644 --- a/src/hermes/model/types/ld_list.py +++ b/src/hermes/model/types/ld_list.py @@ -15,6 +15,7 @@ from typing_extensions import Self from .ld_container import ( + COMPACTED_JSON_LD_VALUE, ld_container, JSON_LD_CONTEXT_DICT, EXPANDED_JSON_LD_VALUE, @@ -23,6 +24,7 @@ TIME_TYPE, BASIC_TYPE, ) +from .pyld_util import bundled_loader if TYPE_CHECKING: from .ld_dict import ld_dict @@ -548,6 +550,45 @@ def to_python(self: Self) -> list[Union[BASIC_TYPE, TIME_TYPE, PYTHONIZED_LD_CON for item in self ] + def compact( + self: Self, context: Optional[Union[list[Union[JSON_LD_CONTEXT_DICT, str]], JSON_LD_CONTEXT_DICT, str]] = None + ) -> COMPACTED_JSON_LD_VALUE: + """ + Returns the compacted version of the given ld_list using its context only if none was supplied. + The returned object is of the form `{"@context": the_context, container_type: compacted_content}`. + + Args: + context (list[JSON_LD_CONTEXT_DICT | str] | JSON_LD_CONTEXT_DICT | str | None): + The context to use for the compaction. If None the context of self is used. + + Returns: + COMPACTED_JSON_LD_VALUE: The compacted version of selfs JSON-LD representation. + """ + # compact the ld_list standalone if necessary + if self.key is None: + return self.ld_proc.compact( + self.ld_value, context or self.full_context, {"documentLoader": bundled_loader, "skipExpand": True} + ) + # compact the ld_list within a temporary dictionary + temp_dict = self.ld_proc.compact( + [{self.ld_proc.expand_iri(self.active_ctx, self.key): self.ld_value}], + context or self.full_context, + {"documentLoader": bundled_loader, "skipExpand": True} + ) + context = temp_dict["@context"] + temp_container = temp_dict[ + self.ld_proc.compact_iri(self.active_ctx, self.ld_proc.expand_iri(self.active_ctx, self.key)) + ] + if self.container_type != "@set": + return { + "@context": context, + **temp_container + } + return { + "@context": context, + "@set": temp_container if isinstance(temp_container, list) else [temp_container] + } + @classmethod def is_ld_list(cls: type[Self], ld_value: Any) -> bool: """ From 4686353f5fda49ca3c5434b2e904b51f8690d6de Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Thu, 21 May 2026 13:25:18 +0200 Subject: [PATCH 05/41] fixed provenance document loading in harvest --- src/hermes/commands/harvest/base.py | 5 ++--- src/hermes/model/provenance/ld_prov.py | 19 +++++++++++++++++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index e6fbfc4c..30c00f23 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -139,7 +139,7 @@ def __call__(self, args: argparse.Namespace) -> None: }) with ctx["provenance"] as cache: - cache["codemeta"] = prov_doc.ld_value + cache["result"] = prov_doc.ld_value ctx.finalize_step('harvest') if not harvested_any: @@ -151,8 +151,7 @@ def init_provenance_document(self) -> ld_prov_list: ctx.prepare_step("harvest") with ctx["provenance"] as cache: try: - ld_prov_doc = ld_prov_list.from_list(cache["codemeta"], container_type="@graph", context=ALL_CONTEXTS) - return ld_prov_doc + return ld_prov_list.load_ld_prov_list(cache["result"]) except KeyError: pass prov_doc = ld_prov_list() diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index df4b8788..4ce93a4b 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -25,14 +25,29 @@ class ld_prov_list(ld_list): def __init__( self: Self, - *, data: EXPANDED_JSON_LD_VALUE = [{"@graph": []}], + *, parent: Optional[Union[ld_dict, ld_list]] = None, key: Optional[str] = PROV_DOC_IRI, index: Optional[int] = None, context: Optional[list[Union[str, JSON_LD_CONTEXT_DICT]]] = ALL_CONTEXTS ) -> None: - super().__init__([{"@graph": []}], parent=parent, key=key, index=index, context=context) + super().__init__(data, parent=parent, key=key, index=index, context=context) + + @classmethod + def load_ld_prov_list(cls, data) -> "ld_prov_list": + if cls.INDICES != {}: + raise RuntimeError("Only zero or one objects of class 'ld_prov_list' may exist at every point in time.") + prov_list = cls.from_list(data[0]["@graph"], container_type="@graph", context=ALL_CONTEXTS, key=cls.PROV_DOC_IRI) + for item in prov_list: + if not ("@id" in item and item["@id"].startswith("_:")): + continue + item_id = item["@id"][2:].split("/") + if not (len(item_id) == 2 and item_id[1].isnumeric()): + continue + if cls.INDICES.get(item_id[0], 0) < int(item_id[1]): + cls.INDICES[item_id[0]] = int(item_id[1]) + return prov_list def next_node_iri(self, type) -> str: if type not in ld_prov_list.INDICES: From c05601b7753a7cb6619983475c87ca602879d723 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Thu, 21 May 2026 13:29:33 +0200 Subject: [PATCH 06/41] flake8 --- src/hermes/commands/harvest/base.py | 1 - src/hermes/model/provenance/ld_prov.py | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index 30c00f23..f1ae63b7 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -13,7 +13,6 @@ from hermes.model.context_manager import HermesContext from hermes.model import SoftwareMetadata from hermes.model.provenance.ld_prov import ld_prov_list -from hermes.model.types.ld_context import ALL_CONTEXTS class HermesHarvestPlugin(HermesPlugin): diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index 4ce93a4b..3c6ea7c3 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -38,7 +38,9 @@ def __init__( def load_ld_prov_list(cls, data) -> "ld_prov_list": if cls.INDICES != {}: raise RuntimeError("Only zero or one objects of class 'ld_prov_list' may exist at every point in time.") - prov_list = cls.from_list(data[0]["@graph"], container_type="@graph", context=ALL_CONTEXTS, key=cls.PROV_DOC_IRI) + prov_list = cls.from_list( + data[0]["@graph"], key=cls.PROV_DOC_IRI, context=ALL_CONTEXTS, container_type="@graph" + ) for item in prov_list: if not ("@id" in item and item["@id"].startswith("_:")): continue From 379fe0fbd90731f0c288f3ab85ac08627c13d0b4 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Thu, 21 May 2026 16:31:45 +0200 Subject: [PATCH 07/41] first draft of provenance recording for process step --- src/hermes/commands/harvest/base.py | 18 ++-- src/hermes/commands/process/base.py | 112 ++++++++++++++++++++++++- src/hermes/model/provenance/ld_prov.py | 12 +-- 3 files changed, 128 insertions(+), 14 deletions(-) diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index f1ae63b7..51744a50 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -128,13 +128,19 @@ def __call__(self, args: argparse.Namespace) -> None: }) # TODO: add more info prov_doc.add_entity(data={ - "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": data_output.ref, + "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref }) prov_doc.add_entity(data={ - "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": data_output.ref, + "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref }) prov_doc.add_entity(data={ - "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": data_output.ref, + "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref }) with ctx["provenance"] as cache: @@ -162,7 +168,7 @@ def remove_provenance_info_for_plugin(self, prov_doc: ld_prov_list, plugin) -> N if plugin is None: return # two passes are needed because the nodes are nested exactly two levels - related = prov_doc.shallow_search(lambda doc, node: ( + related = prov_doc.shallow_search(lambda node: ( ("prov:wasAssociatedWith" in node and plugin.ref in node["prov:wasAssociatedWith"]) or ("prov:wasAttributedTo" in node and plugin.ref in node["prov:wasAttributedTo"]) )) @@ -170,11 +176,11 @@ def remove_provenance_info_for_plugin(self, prov_doc: ld_prov_list, plugin) -> N del prov_doc[plugin.index] return ids = [plugin.ref, *(rel.ref for rel in related)] - related = prov_doc.shallow_search(lambda doc, node: any( + related = prov_doc.shallow_search(lambda node: any( (f"prov:{key}" in node and id in node[f"prov:{key}"]) for id in ids for key in [ "wasAssociatedWith", "wasAttributedTo", "wasGeneratedBy", "used", "wasDerivedFrom", "wasInformedBy" ] )) for item in related: - items = prov_doc.shallow_search(lambda doc, node: ("@id" in node and node["@id"] == item["@id"])) + items = prov_doc.shallow_search(lambda node: ("@id" in node and node["@id"] == item["@id"])) del prov_doc[items[0].index] diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index 725f6487..a603c650 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -5,7 +5,7 @@ # SPDX-FileContributor: Michael Meinel import argparse -from typing import Union +from typing import Optional from pydantic import BaseModel @@ -15,12 +15,13 @@ from hermes.model.context_manager import HermesContext from hermes.model.merge.action import MergeAction from hermes.model.merge.container import ld_merge_dict +from hermes.model.provenance.ld_prov import ld_prov_list class HermesProcessPlugin(HermesPlugin): """ Base plugin that defines additional merge strategies.""" - def __call__(self, command: HermesCommand) -> dict[Union[str, None], dict[Union[str, None], MergeAction]]: + def __call__(self, command: HermesCommand) -> dict[Optional[str], dict[Optional[str], MergeAction]]: pass @@ -38,6 +39,12 @@ class HermesProcessCommand(HermesCommand): settings_class = ProcessSettings def __call__(self, args: argparse.Namespace) -> None: + self.log.info("# Load provenance data from harvest step") + prov_doc = self.load_prov_doc() + if prov_doc is not None: + process_command = prov_doc.get_hermes_command("process") + hermes_cache = prov_doc.get_hermes_cache() + self.log.info("# Metadata processing") merged_doc = ld_merge_dict([{}]) @@ -56,6 +63,7 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info("## Load and run the plugins") any_strategies_loaded = False + strategy_action, merged_strategies = None, None # add the strategies from the plugins for plugin_name in reversed(self.settings.plugins): self.log.info(f"### Load {plugin_name} plugin") @@ -79,6 +87,28 @@ def __call__(self, args: argparse.Namespace) -> None: merged_doc.add_strategy(additional_strategies) any_strategies_loaded = True + if prov_doc is None: + continue + plugin = prov_doc.add_hermes_plugin("process", plugin_name) + new_strategy_generation = prov_doc.add_activity(data={"prov:wasAssociatedWith": plugin.ref}) + new_strategies = prov_doc.add_entity( + data={"prov:wasAttributedTo": plugin.ref, "prov:wasGeneratedBy": new_strategy_generation.ref} + ) + if merged_strategies is None: + merged_strategies = new_strategies + strategy_action = new_strategy_generation + continue + strategy_action = prov_doc.add_activity(data={ + "prov:used": [merged_strategies.ref, new_strategies.ref], + "prov:wasInformedBy": [strategy_action.ref, new_strategy_generation.ref], + "prov:wasAssociatedWith": process_command.ref + }) + merged_strategies = prov_doc.add_entity(data={ + "prov:wasDerivedFrom": [merged_strategies.ref, new_strategies.ref], + "prov:wasGeneratedBy": strategy_action.ref, + "prov:wasAttributedTo": process_command.ref + }) + if not any_strategies_loaded: self.log.critical("## No process plugin was ran successfully.") raise HermesPluginRunError("No process plugin was ran successfully.") @@ -89,6 +119,7 @@ def __call__(self, args: argparse.Namespace) -> None: # merge data from harvesters self.log.info("## Merge the metadata of the harvesters") merged_any = False + merge_action, merged_data = None, None for harvester in harvester_names: self.log.info(f"### Load data from {harvester} plugin") # load data from harvester @@ -110,6 +141,43 @@ def __call__(self, args: argparse.Namespace) -> None: raise RuntimeError(f"Merging the data from {harvester} plugin failed.") from e merged_any = True + if prov_doc is None: + continue + harvest_plugin = prov_doc.get_hermes_plugin("harvest", harvester) + harvest_command = prov_doc.get_hermes_command("harvest") + store_action = prov_doc.shallow_search(lambda node: ( + "prov:wasAssociatedWith" in node and + node["prov:wasAssociatedWith"] == [harvest_plugin.ref, hermes_cache.ref, harvest_command.ref] + ))[0] + stored_results = [ + result.ref for result in prov_doc.shallow_search( + lambda node: ("prov:wasGeneratedBy" in node and node["prov:wasGeneratedBy"] == store_action.ref) + ) + ] + new_data_load = prov_doc.add_activity(data={ + "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref], + "prov:used": stored_results + }) + new_data = prov_doc.add_entity(data={ + "prov:wasAttributedTo": plugin.ref, + "prov:wasGeneratedBy": new_data_load.ref, + "prov:wasDerivedFrom": stored_results + }) + if merged_data is None: + merged_data = new_data + merge_action = new_data_load + continue + merge_action = prov_doc.add_activity(data={ + "prov:used": [merged_data.ref, new_data.ref, merged_strategies.ref], + "prov:wasInformedBy": [merge_action.ref, new_data_load.ref], + "prov:wasAssociatedWith": process_command.ref + }) + merged_data = prov_doc.add_entity(data={ + "prov:wasDerivedFrom": [merged_data.ref, new_data.ref], + "prov:wasGeneratedBy": merge_action.ref, + "prov:wasAttributedTo": process_command.ref + }) + # error if nothing was merged if not merged_any: self.log.critical("No metadata has been merged, the loading of the data failed for all harvesters.") @@ -122,6 +190,46 @@ def __call__(self, args: argparse.Namespace) -> None: result_ctx["codemeta"] = merged_doc.compact() result_ctx["context"] = {"@context": merged_doc.full_context} result_ctx["expanded"] = merged_doc.ld_value + + if prov_doc is not None: + write = prov_doc.add_activity(data={ + "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref, plugin.ref], + "prov:used": merged_data.ref, + "prov:wasInformedBy": merge_action.ref + }) + # TODO: add more info + prov_doc.add_entity(data={ + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": merged_data.ref, + "prov:wasAttributedTo": hermes_cache.ref + }) + prov_doc.add_entity(data={ + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": merged_data.ref, + "prov:wasAttributedTo": hermes_cache.ref + }) + prov_doc.add_entity(data={ + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": merged_data.ref, + "prov:wasAttributedTo": hermes_cache.ref + }) + + with ctx["provenance"] as cache: + cache["result"] = prov_doc.ld_value + ctx.finalize_step("process") ctx.finalize_step("harvest") + + def load_prov_doc(self) -> Optional[ld_prov_list]: + ctx = HermesContext() + ctx.prepare_step("harvest") + with ctx["provenance"] as cache: + try: + return ld_prov_list.load_ld_prov_list(cache["result"]) + except Exception: + self.log.warning( + "The provenance data from the harvest step could not be loaded." + "Processing will proceed without collecting provenance data.", + exc_info=1 + ) diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index 3c6ea7c3..9edc4019 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -132,21 +132,21 @@ def add_hermes_plugin(self, step, name) -> ld_dict: return node def shallow_search(self, query) -> list[ld_dict]: - return [item for item in self if query(self, item)] + return [item for item in self if query(item)] def get_hermes(self) -> ld_dict: - return self.shallow_search(lambda doc, node: ("@id" in node and node["@id"] == ld_prov_list.HERMES_ID))[0] + return self.shallow_search(lambda node: ("@id" in node and node["@id"] == ld_prov_list.HERMES_ID))[0] def get_hermes_cache(self) -> ld_dict: - return self.shallow_search(lambda doc, node: ("@id" in node and node["@id"] == ld_prov_list.HERMES_CACHE_ID))[0] + return self.shallow_search(lambda node: ("@id" in node and node["@id"] == ld_prov_list.HERMES_CACHE_ID))[0] def get_hermes_base_plugin(self, step) -> ld_dict: - return self.shallow_search(lambda doc, node: ( + return self.shallow_search(lambda node: ( "@id" in node and node["@id"] == ld_prov_list.HERMES_BASE_PLUGIN_ID_FORMAT.format(step=step) ))[0] def get_hermes_plugin(self, step, name) -> Union[ld_dict, None]: - search_result = self.shallow_search(lambda doc, node: ( + search_result = self.shallow_search(lambda node: ( "@id" in node and node["@id"] == ld_prov_list.HERMES_PLUGIN_ID_FORMAT.format(step=step, name=name) )) if search_result: @@ -154,6 +154,6 @@ def get_hermes_plugin(self, step, name) -> Union[ld_dict, None]: return None def get_hermes_command(self, step) -> ld_dict: - return self.shallow_search(lambda doc, node: ( + return self.shallow_search(lambda node: ( "@id" in node and node["@id"] == ld_prov_list.HERMES_COMMAND_ID_FORMAT.format(step=step) ))[0] From 40a848ab1abccd0ca1e6a5c1fce0d8c6b7a3a68e Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Thu, 28 May 2026 14:13:28 +0200 Subject: [PATCH 08/41] improved provenance collection for harvest --- src/hermes/commands/harvest/base.py | 47 +++++++++++++++++++++----- src/hermes/commands/process/base.py | 2 +- src/hermes/model/provenance/ld_prov.py | 12 +++---- 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index 51744a50..f720f8b7 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -5,6 +5,7 @@ # SPDX-FileContributor: Michael Meinel import argparse +import datetime from pydantic import BaseModel @@ -28,9 +29,11 @@ def __call__(self, command: HermesCommand) -> SoftwareMetadata: pass def load(): + # TODO: Implement pass def write(): + # TODO: Implement pass @@ -75,14 +78,17 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info(f"### Run {plugin_name} plugin") # run plugin try: - harvested_data = plugin_func(self) + harvested_data: SoftwareMetadata = plugin_func(self) except Exception: self.log.exception(f"### Unknown error while executing the {plugin_name} plugin, skipping it now.") continue + returned_at_time = datetime.datetime.now().isoformat() self.log.info(f"### Store metadata harvested by {plugin_name} plugin") # store harvested data + begin_store_at_time = datetime.datetime.now().isoformat() harvested_data.write_to_cache(ctx, plugin_name) + stored_at_time = datetime.datetime.now().isoformat() harvested_any = True self.remove_provenance_info_for_plugin(prov_doc, plugin_name) @@ -107,40 +113,64 @@ def __call__(self, args: argparse.Namespace) -> None: io_ops.append(io_op.ref) map_activity = prov_doc.add_activity(data={ + "schema:description": "Maps the loaded data to the JSON-LD contexts vocabulary.", "prov:wasInformedBy": io_ops, "prov:used": outputs, - "prov:wasAssociatedWith": plugin.ref + "prov:wasAssociatedWith": plugin.ref, + "prov:startedAtTime": returned_at_time }) data_output = prov_doc.add_entity(data={ + "schema:description": "the harvested metadata", "prov:wasAttributedTo": plugin.ref, "prov:wasGeneratedBy": map_activity.ref, - "prov:wasDerivedFrom": outputs + "prov:wasDerivedFrom": outputs, + "prov:generatedAtTime": returned_at_time }) write = prov_doc.add_activity(data={ + "schema:description": "Writes the harvested metadata into the HERMES cache.", "prov:wasAssociatedWith": [ prov_doc.get_hermes_command("harvest").ref, prov_doc.get_hermes_cache().ref, plugin.ref ], "prov:used": data_output.ref, - "prov:wasInformedBy": map_activity.ref + "prov:wasInformedBy": map_activity.ref, + "prov:startedAtTime": begin_store_at_time, + "prov:endedAtTime": stored_at_time }) - # TODO: add more info prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The compacted version of the harvested metadata.", + "schema:text": str(harvested_data.compact()), + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "codemeta.json").as_uri(), "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref, - "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref + "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref, + "prov:generatedAtTime": stored_at_time }) prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The context of the harvested metadata.", + "schema:text": str({"@context": harvested_data.full_context}), + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "context.json").as_uri(), "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref, - "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref + "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref, + "prov:generatedAtTime": stored_at_time }) prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The expanded version of the harvested metadata.", + "schema:text": str(harvested_data.ld_value), + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "expanded.json").as_uri(), "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref, - "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref + "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref, + "prov:generatedAtTime": stored_at_time }) with ctx["provenance"] as cache: @@ -181,6 +211,7 @@ def remove_provenance_info_for_plugin(self, prov_doc: ld_prov_list, plugin) -> N "wasAssociatedWith", "wasAttributedTo", "wasGeneratedBy", "used", "wasDerivedFrom", "wasInformedBy" ] )) + del prov_doc[plugin.index] for item in related: items = prov_doc.shallow_search(lambda node: ("@id" in node and node["@id"] == item["@id"])) del prov_doc[items[0].index] diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index a603c650..51bac589 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -229,7 +229,7 @@ def load_prov_doc(self) -> Optional[ld_prov_list]: return ld_prov_list.load_ld_prov_list(cache["result"]) except Exception: self.log.warning( - "The provenance data from the harvest step could not be loaded." + "The provenance data from the harvest step could not be loaded. " "Processing will proceed without collecting provenance data.", exc_info=1 ) diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index 9edc4019..6604df1c 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -61,9 +61,9 @@ def add_activity(self, *, data={}) -> ld_dict: self.append(data) activity = self[-1] if "@type" not in data: - activity["@type"] = "prov:Activity" + activity["@type"] = ["prov:Activity", "schema:Action"] else: - activity["@type"].append("prov:Activity") + activity["@type"].extend(["prov:Activity", "schema:Action"]) if "@id" not in data: activity["@id"] = self.next_node_iri("Activity") return activity @@ -72,9 +72,9 @@ def add_agent(self, *, data={}) -> ld_dict: self.append(data) agent = self[-1] if "@type" not in data: - agent["@type"] = "prov:Agent" + agent["@type"] = ["prov:Agent", "schema:SoftwareApplication"] else: - agent["@type"].append("prov:Agent") + agent["@type"].extend(["prov:Agent", "schema:SoftwareApplication"]) if "@id" not in data: agent["@id"] = self.next_node_iri("Agent") return agent @@ -83,9 +83,9 @@ def add_entity(self, *, data={}) -> ld_dict: self.append(data) entity = self[-1] if "@type" not in data: - entity["@type"] = "prov:Entity" + entity["@type"] = ["prov:Entity", "schema:Thing"] else: - entity["@type"].append("prov:Entity") + entity["@type"].extend(["prov:Entity", "schema:Thing"]) if "@id" not in data: entity["@id"] = self.next_node_iri("Entity") return entity From d637f84832f6bf2944ce2a731ca8a1d34ba75c2b Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Mon, 1 Jun 2026 16:26:14 +0200 Subject: [PATCH 09/41] implement load wrapper for harvest plugins and fixed minor bug --- src/hermes/commands/harvest/base.py | 39 ++++++++++++++++++------- src/hermes/commands/harvest/cff.py | 2 +- src/hermes/commands/harvest/codemeta.py | 2 +- 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index f720f8b7..7545c52f 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -6,6 +6,8 @@ import argparse import datetime +from io import IOBase +from pathlib import Path from pydantic import BaseModel @@ -28,9 +30,24 @@ def __init__(self): def __call__(self, command: HermesCommand) -> SoftwareMetadata: pass - def load(): - # TODO: Implement - pass + def load(self, func, source, *args, **kwargs): + source_metadata = {"schema:description": "metadata source"} + if isinstance(source, IOBase): + source_metadata["schema:url"] = Path(source.name).absolute().as_uri() + elif isinstance(source, Path): + source_metadata["schema:url"] = source.absolute().as_uri() + elif isinstance(source, str): + source_metadata["schema:url"] = Path(source).absolute().as_uri() + io_operation = { + "schema:description": "Load operation called with (" + f"{source_metadata['schema:url'] if 'schema:url' in source_metadata else str(source)}" + f"{', ' + str(args) if args else ''}{', ' + str(kwargs) if kwargs else ''}).", + "schema:name": f"{func.__module__}.{func.__qualname__}" + } + result = func(source, *args, **kwargs) + loaded_metadata = {"schema:description": "the loaded data", "schema:text": str(result)} + self.io_operations.append((source_metadata, io_operation, loaded_metadata)) + return result def write(): # TODO: Implement @@ -108,7 +125,7 @@ def __call__(self, args: argparse.Namespace) -> None: "prov:wasDerivedFrom": loaded_source.ref, "prov:wasGeneratedBy": io_op.ref }) - loaded_data = prov_doc.add_entity(data=plugin_io_operations[2]) + loaded_data = prov_doc.add_entity(data=plugin_io_operation[2]) outputs.append(loaded_data.ref) io_ops.append(io_op.ref) @@ -144,7 +161,7 @@ def __call__(self, args: argparse.Namespace) -> None: "schema:description": "The compacted version of the harvested metadata.", "schema:text": str(harvested_data.compact()), "schema:encodingFormat": "application/json", - "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "codemeta.json").as_uri(), + "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "codemeta.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref, "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref, @@ -155,7 +172,7 @@ def __call__(self, args: argparse.Namespace) -> None: "schema:description": "The context of the harvested metadata.", "schema:text": str({"@context": harvested_data.full_context}), "schema:encodingFormat": "application/json", - "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "context.json").as_uri(), + "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "context.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref, "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref, @@ -166,7 +183,7 @@ def __call__(self, args: argparse.Namespace) -> None: "schema:description": "The expanded version of the harvested metadata.", "schema:text": str(harvested_data.ld_value), "schema:encodingFormat": "application/json", - "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "expanded.json").as_uri(), + "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "expanded.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref, "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref, @@ -197,7 +214,6 @@ def remove_provenance_info_for_plugin(self, prov_doc: ld_prov_list, plugin) -> N plugin = prov_doc.get_hermes_plugin("harvest", plugin) if plugin is None: return - # two passes are needed because the nodes are nested exactly two levels related = prov_doc.shallow_search(lambda node: ( ("prov:wasAssociatedWith" in node and plugin.ref in node["prov:wasAssociatedWith"]) or ("prov:wasAttributedTo" in node and plugin.ref in node["prov:wasAttributedTo"]) @@ -206,7 +222,9 @@ def remove_provenance_info_for_plugin(self, prov_doc: ld_prov_list, plugin) -> N del prov_doc[plugin.index] return ids = [plugin.ref, *(rel.ref for rel in related)] - related = prov_doc.shallow_search(lambda node: any( + used_entities = [rel["prov:used"][0]["@id"] for rel in related if "prov:used" in rel] + related = prov_doc.shallow_search(lambda node: node["@id"] in used_entities) + related += prov_doc.shallow_search(lambda node: any( (f"prov:{key}" in node and id in node[f"prov:{key}"]) for id in ids for key in [ "wasAssociatedWith", "wasAttributedTo", "wasGeneratedBy", "used", "wasDerivedFrom", "wasInformedBy" ] @@ -214,4 +232,5 @@ def remove_provenance_info_for_plugin(self, prov_doc: ld_prov_list, plugin) -> N del prov_doc[plugin.index] for item in related: items = prov_doc.shallow_search(lambda node: ("@id" in node and node["@id"] == item["@id"])) - del prov_doc[items[0].index] + if len(items) == 1: + del prov_doc[items[0].index] diff --git a/src/hermes/commands/harvest/cff.py b/src/hermes/commands/harvest/cff.py index 5a2d16c1..f2b648e6 100644 --- a/src/hermes/commands/harvest/cff.py +++ b/src/hermes/commands/harvest/cff.py @@ -43,7 +43,7 @@ def __call__(self, command: HermesHarvestCommand) -> tuple[SoftwareMetadata, dic 'Aborting harvesting for this metadata source.') # Read the content - cff_data = cff_file.read_text() + cff_data = self.load(pathlib.Path.read_text, cff_file) cff_dict = self._load_cff_from_file(cff_data) if command.settings.cff.enable_validation: diff --git a/src/hermes/commands/harvest/codemeta.py b/src/hermes/commands/harvest/codemeta.py index 3dc84296..07645647 100644 --- a/src/hermes/commands/harvest/codemeta.py +++ b/src/hermes/commands/harvest/codemeta.py @@ -34,7 +34,7 @@ def __call__(self, command: HermesHarvestCommand) -> tuple[SoftwareMetadata, dic ) # Read the content - codemeta_str = codemeta_file.read_text() + codemeta_str = self.load(pathlib.Path.read_text, codemeta_file) if not self._validate(codemeta_file): raise HermesValidationError(codemeta_file) From 4d8e62da8d8ba024e7dc024318c972169ec2e9fc Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Mon, 8 Jun 2026 17:34:38 +0200 Subject: [PATCH 10/41] implemented draft of provenance collection druing merge --- src/hermes/commands/harvest/base.py | 65 +++++++++---------- src/hermes/commands/process/base.py | 88 +++++++++++++------------- src/hermes/model/merge/container.py | 51 ++++++++++++++- src/hermes/model/provenance/ld_prov.py | 2 +- 4 files changed, 127 insertions(+), 79 deletions(-) diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index 7545c52f..56799e9c 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -38,10 +38,10 @@ def load(self, func, source, *args, **kwargs): source_metadata["schema:url"] = source.absolute().as_uri() elif isinstance(source, str): source_metadata["schema:url"] = Path(source).absolute().as_uri() - io_operation = { + io_operation = { "schema:description": "Load operation called with (" - f"{source_metadata['schema:url'] if 'schema:url' in source_metadata else str(source)}" - f"{', ' + str(args) if args else ''}{', ' + str(kwargs) if kwargs else ''}).", + f"{source_metadata['schema:url'] if 'schema:url' in source_metadata else str(source)}" + f"{', ' + str(args) if args else ''}{', ' + str(kwargs) if kwargs else ''}).", "schema:name": f"{func.__module__}.{func.__qualname__}" } result = func(source, *args, **kwargs) @@ -60,6 +60,32 @@ class HarvestSettings(BaseModel): sources: list[str] = [] +def remove_harvest_plugin_from_prov_doc(prov_doc: ld_prov_list, plugin: str) -> None: + plugin = prov_doc.get_hermes_plugin("harvest", plugin) + if plugin is None: + return + related = prov_doc.shallow_search(lambda node: ( + ("prov:wasAssociatedWith" in node and plugin.ref in node["prov:wasAssociatedWith"]) or + ("prov:wasAttributedTo" in node and plugin.ref in node["prov:wasAttributedTo"]) + )) + if len(related) == 0: + del prov_doc[plugin.index] + return + ids = [plugin.ref, *(rel.ref for rel in related)] + used_entities = [rel["prov:used"][0]["@id"] for rel in related if "prov:used" in rel] + related = prov_doc.shallow_search(lambda node: node["@id"] in used_entities) + related += prov_doc.shallow_search(lambda node: any( + (f"prov:{key}" in node and id in node[f"prov:{key}"]) for id in ids for key in [ + "wasAssociatedWith", "wasAttributedTo", "wasGeneratedBy", "used", "wasDerivedFrom", "wasInformedBy" + ] + )) + del prov_doc[plugin.index] + for item in related: + items = prov_doc.shallow_search(lambda node: ("@id" in node and node["@id"] == item["@id"])) + if len(items) == 1: + del prov_doc[items[0].index] + + class HermesHarvestCommand(HermesCommand): """ Harvest metadata from configured sources. """ @@ -108,7 +134,7 @@ def __call__(self, args: argparse.Namespace) -> None: stored_at_time = datetime.datetime.now().isoformat() harvested_any = True - self.remove_provenance_info_for_plugin(prov_doc, plugin_name) + remove_harvest_plugin_from_prov_doc(prov_doc, plugin_name) plugin = prov_doc.add_hermes_plugin("harvest", plugin_name) plugin_io_operations = plugin_func.io_operations @@ -159,7 +185,7 @@ def __call__(self, args: argparse.Namespace) -> None: prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "The compacted version of the harvested metadata.", - "schema:text": str(harvested_data.compact()), + "schema:text": str(harvested_data.compact()), # TODO: maybe "prov:value" instead? "schema:encodingFormat": "application/json", "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "codemeta.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, @@ -170,7 +196,7 @@ def __call__(self, args: argparse.Namespace) -> None: prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "The context of the harvested metadata.", - "schema:text": str({"@context": harvested_data.full_context}), + "schema:text": str({"@context": harvested_data.full_context}), # TODO: maybe "prov:value" instead? "schema:encodingFormat": "application/json", "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "context.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, @@ -181,7 +207,7 @@ def __call__(self, args: argparse.Namespace) -> None: prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "The expanded version of the harvested metadata.", - "schema:text": str(harvested_data.ld_value), + "schema:text": str(harvested_data.ld_value), # TODO: maybe "prov:value" instead? "schema:encodingFormat": "application/json", "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "expanded.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, @@ -209,28 +235,3 @@ def init_provenance_document(self) -> ld_prov_list: prov_doc = ld_prov_list() prov_doc.init_hermes_agents() return prov_doc - - def remove_provenance_info_for_plugin(self, prov_doc: ld_prov_list, plugin) -> None: - plugin = prov_doc.get_hermes_plugin("harvest", plugin) - if plugin is None: - return - related = prov_doc.shallow_search(lambda node: ( - ("prov:wasAssociatedWith" in node and plugin.ref in node["prov:wasAssociatedWith"]) or - ("prov:wasAttributedTo" in node and plugin.ref in node["prov:wasAttributedTo"]) - )) - if len(related) == 0: - del prov_doc[plugin.index] - return - ids = [plugin.ref, *(rel.ref for rel in related)] - used_entities = [rel["prov:used"][0]["@id"] for rel in related if "prov:used" in rel] - related = prov_doc.shallow_search(lambda node: node["@id"] in used_entities) - related += prov_doc.shallow_search(lambda node: any( - (f"prov:{key}" in node and id in node[f"prov:{key}"]) for id in ids for key in [ - "wasAssociatedWith", "wasAttributedTo", "wasGeneratedBy", "used", "wasDerivedFrom", "wasInformedBy" - ] - )) - del prov_doc[plugin.index] - for item in related: - items = prov_doc.shallow_search(lambda node: ("@id" in node and node["@id"] == item["@id"])) - if len(items) == 1: - del prov_doc[items[0].index] diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index 51bac589..5711b197 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -10,6 +10,7 @@ from pydantic import BaseModel from hermes.commands.base import HermesCommand, HermesPlugin +from hermes.commands.harvest.base import remove_harvest_plugin_from_prov_doc from hermes.error import HermesPluginRunError, MisconfigurationError from hermes.model.api import SoftwareMetadata from hermes.model.context_manager import HermesContext @@ -46,7 +47,7 @@ def __call__(self, args: argparse.Namespace) -> None: hermes_cache = prov_doc.get_hermes_cache() self.log.info("# Metadata processing") - merged_doc = ld_merge_dict([{}]) + merged_doc = ld_merge_dict([{}], prov_doc) if not self.settings.plugins: self.log.critical( @@ -119,7 +120,6 @@ def __call__(self, args: argparse.Namespace) -> None: # merge data from harvesters self.log.info("## Merge the metadata of the harvesters") merged_any = False - merge_action, merged_data = None, None for harvester in harvester_names: self.log.info(f"### Load data from {harvester} plugin") # load data from harvester @@ -127,56 +127,56 @@ def __call__(self, args: argparse.Namespace) -> None: metadata = SoftwareMetadata.load_from_cache(ctx, harvester) except Exception: # skip this harvester when the data is invalid + if prov_doc is not None: + remove_harvest_plugin_from_prov_doc(prov_doc, harvester) self.log.exception( f"### The data from the harvester {harvester} could not be loaded or is invalid, skipping it now." ) continue + if prov_doc is not None: + harvest_plugin = prov_doc.get_hermes_plugin("harvest", harvester) + harvest_command = prov_doc.get_hermes_command("harvest") + store_action = prov_doc.shallow_search(lambda node: ( + "prov:wasAssociatedWith" in node and + node["prov:wasAssociatedWith"] == [harvest_plugin.ref, hermes_cache.ref, harvest_command.ref] + ))[0] + stored_results = [ + result.ref for result in prov_doc.shallow_search(lambda node: ( + "prov:wasGeneratedBy" in node and node["prov:wasGeneratedBy"] == [store_action.ref] + )) + ] + new_action = prov_doc.add_activity(data={ # load of new data + "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref], + "prov:used": stored_results + }) + new_data = prov_doc.add_entity(data={ # new data to be merged + "prov:wasAttributedTo": plugin.ref, + "prov:wasGeneratedBy": new_action.ref, + "prov:wasDerivedFrom": stored_results + }) + if merged_any: + # One pass must have been completed already. + new_action = prov_doc.add_activity(data={ + "prov:used": [last_data.ref, new_data.ref], + "prov:wasInformedBy": [last_action.ref, new_action.ref], + "prov:wasAssociatedWith": process_command.ref + }) # initial merge action of the merge + merged_doc.prov_objects = [new_action, new_data, last_data] + self.log.info(f"### Merge data from {harvester} plugin") # merge data into the merge dict try: merged_doc.update(metadata) except Exception as e: + # TODO: Maybe this state is recoverable by starting over again and skipping this plugin. self.log.critical(f"### Merging the data from {harvester} plugin resulted in an error.", exc_info=True) raise RuntimeError(f"Merging the data from {harvester} plugin failed.") from e - merged_any = True - if prov_doc is None: - continue - harvest_plugin = prov_doc.get_hermes_plugin("harvest", harvester) - harvest_command = prov_doc.get_hermes_command("harvest") - store_action = prov_doc.shallow_search(lambda node: ( - "prov:wasAssociatedWith" in node and - node["prov:wasAssociatedWith"] == [harvest_plugin.ref, hermes_cache.ref, harvest_command.ref] - ))[0] - stored_results = [ - result.ref for result in prov_doc.shallow_search( - lambda node: ("prov:wasGeneratedBy" in node and node["prov:wasGeneratedBy"] == store_action.ref) - ) - ] - new_data_load = prov_doc.add_activity(data={ - "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref], - "prov:used": stored_results - }) - new_data = prov_doc.add_entity(data={ - "prov:wasAttributedTo": plugin.ref, - "prov:wasGeneratedBy": new_data_load.ref, - "prov:wasDerivedFrom": stored_results - }) - if merged_data is None: - merged_data = new_data - merge_action = new_data_load - continue - merge_action = prov_doc.add_activity(data={ - "prov:used": [merged_data.ref, new_data.ref, merged_strategies.ref], - "prov:wasInformedBy": [merge_action.ref, new_data_load.ref], - "prov:wasAssociatedWith": process_command.ref - }) - merged_data = prov_doc.add_entity(data={ - "prov:wasDerivedFrom": [merged_data.ref, new_data.ref], - "prov:wasGeneratedBy": merge_action.ref, - "prov:wasAttributedTo": process_command.ref - }) + if prov_doc is not None: + last_action = merged_doc.prov_objects[0] if merged_any else new_action + last_data = merged_doc.prov_objects[2] if merged_any else new_data + merged_any = True # error if nothing was merged if not merged_any: @@ -194,23 +194,23 @@ def __call__(self, args: argparse.Namespace) -> None: if prov_doc is not None: write = prov_doc.add_activity(data={ "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref, plugin.ref], - "prov:used": merged_data.ref, - "prov:wasInformedBy": merge_action.ref + "prov:used": last_data.ref, + "prov:wasInformedBy": last_action.ref }) # TODO: add more info prov_doc.add_entity(data={ "prov:wasGeneratedBy": write.ref, - "prov:wasDerivedFrom": merged_data.ref, + "prov:wasDerivedFrom": last_data.ref, "prov:wasAttributedTo": hermes_cache.ref }) prov_doc.add_entity(data={ "prov:wasGeneratedBy": write.ref, - "prov:wasDerivedFrom": merged_data.ref, + "prov:wasDerivedFrom": last_data.ref, "prov:wasAttributedTo": hermes_cache.ref }) prov_doc.add_entity(data={ "prov:wasGeneratedBy": write.ref, - "prov:wasDerivedFrom": merged_data.ref, + "prov:wasDerivedFrom": last_data.ref, "prov:wasAttributedTo": hermes_cache.ref }) diff --git a/src/hermes/model/merge/container.py b/src/hermes/model/merge/container.py index e9cc03a2..a286c7e8 100644 --- a/src/hermes/model/merge/container.py +++ b/src/hermes/model/merge/container.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Callable, Optional, Union from typing_extensions import Self +from hermes.model.provenance.ld_prov import ld_prov_list from hermes.model.types import ld_container, ld_context, ld_dict, ld_list from hermes.model.types.ld_container import ( BASIC_TYPE, EXPANDED_JSON_LD_VALUE, JSON_LD_CONTEXT_DICT, JSON_LD_VALUE, TIME_TYPE @@ -50,6 +51,8 @@ def _to_python( if isinstance(value, ld_dict) and not isinstance(value, ld_merge_dict): value = ld_merge_dict( value.ld_value, + self.prov_doc, + self.prov_objects, parent=value.parent, key=value.key, index=value.index, @@ -60,6 +63,8 @@ def _to_python( if isinstance(value, ld_list) and not isinstance(value, ld_merge_list): value = ld_merge_list( value.ld_value, + self.prov_doc, + self.prov_objects, parent=value.parent, key=value.key, index=value.index, @@ -82,6 +87,8 @@ class ld_merge_list(_ld_merge_container, ld_list): def __init__( self: "ld_merge_list", data: Union[list[str], list[dict[str, EXPANDED_JSON_LD_VALUE]]], + prov_doc: ld_prov_list = None, + prov_objects: list[ld_dict] = 3*[None], *, parent: Optional[ld_container] = None, key: Optional[str] = None, @@ -108,6 +115,8 @@ def __init__( super().__init__(data, parent=parent, key=key, index=index, context=context) self.strategies = strategies + self.prov_doc = prov_doc + self.prov_objects = prov_objects class ld_merge_dict(_ld_merge_container, ld_dict): @@ -123,6 +132,8 @@ class ld_merge_dict(_ld_merge_container, ld_dict): def __init__( self: Self, data: list[dict[str, EXPANDED_JSON_LD_VALUE]], + prov_doc: ld_prov_list = None, + prov_objects: list[ld_dict] = 3*[None], *, parent: Optional[Union[ld_dict, ld_list]] = None, key: Optional[str] = None, @@ -154,6 +165,8 @@ def __init__( # add strategies self.strategies = strategies + self.prov_doc = prov_doc + self.prov_objects = prov_objects def update_context( self: Self, other_context: Union[list[Union[str, JSON_LD_CONTEXT_DICT]], None] @@ -225,9 +238,32 @@ def __setitem__(self: Self, key: str, value: Union[JSON_LD_VALUE, BASIC_TYPE, TI """ # create the new item if self[key] and value have to be merged. if key in self: - value = self._merge_item(key, value) + if self.prov_objects[0] is not None: + last_merged_data = self.prov_objects[2] + merge_activity, value = self._merge_item(key, value) + if self.prov_objects[0] is not None: + create_new_merged_data = last_merged_data is self.prov_objects[2] + elif self.prov_objects[0] is not None: + merge_activity = self.prov_doc.add_activity(data={ + "schema:name": "merge", + "schema:description": "foo", + "prov:used": {"@list": [self.prov_objects[2].ref, str(self.path+[key])]}, + "prov:wasInformedBy": self.prov_objects[0].ref + }) + create_new_merged_data = True # update the entry of self[key] super().__setitem__(key, value) + if self.prov_objects[0] is None: + return + self.prov_objects[0] = merge_activity + if create_new_merged_data: + self.prov_objects[2] = self.prov_doc.add_entity(data={ + "prov:wasAttributedTo": self.prov_doc.get_hermes_command("process").ref, + "prov:wasGeneratedBy": merge_activity.ref, + "prov:wasDerivedFrom": {"@list": [self.prov_objects[1].ref, self.prov_objects[2].ref]} + }) + else: + self.prov_objects[2]["prov:wasGeneratedBy"].append(merge_activity.ref) def match( self: Self, @@ -281,7 +317,18 @@ def _merge_item( merger = strategy.get(key, strategy.get(None, None)) if merger is None: raise MergeError(f"Can't merge, no strategy found for key '{key}'.") - return merger.merge(self, [*self.path, key], self[key], value) + if self.prov_objects[0] is not None: + merge_activity = self.prov_doc.add_activity(data={ + "schema:name": "merge", + "schema:description": "foo", + "prov:wasAssociatedWith": self.prov_doc.get_hermes_command("process").ref, + "prov:used": {"@list": [self.prov_objects[1].ref, self.prov_objects[2].ref, str(self.path+[key])]}, + "prov:wasInformedBy": self.prov_objects[0].ref + }) + self.prov_objects[0] = merge_activity + else: + merge_activity = None + return merge_activity, merger.merge(self, [*self.path, key], self[key], value) def _add_related( self: Self, rel: str, key: str, value: Union[BASIC_TYPE, TIME_TYPE, ld_dict, ld_list] diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index 6604df1c..767d58ad 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -127,7 +127,7 @@ def add_hermes_plugin(self, step, name) -> ld_dict: "@id": ld_prov_list.HERMES_PLUGIN_ID_FORMAT.format(step=step, name=name), "@type": "schema:SoftwareApplication", "schema:name": f"{utils.hermes_name} {step} plugin '{name}'", - "prov:actedOnBehalfOf": self.get_hermes_base_plugin(step) + "prov:actedOnBehalfOf": self.get_hermes_base_plugin(step).ref }) return node From 038b72a50c884944faeaa9019b094302936df301 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Thu, 2 Jul 2026 12:58:45 +0200 Subject: [PATCH 11/41] completed provenance recording in the process step at least for now --- src/hermes/commands/process/base.py | 56 +++++++++++++++++++++++------ src/hermes/model/merge/container.py | 24 +++++++++---- 2 files changed, 64 insertions(+), 16 deletions(-) diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index 5711b197..b7d83a79 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -5,6 +5,7 @@ # SPDX-FileContributor: Michael Meinel import argparse +import datetime from typing import Optional from pydantic import BaseModel @@ -91,20 +92,27 @@ def __call__(self, args: argparse.Namespace) -> None: if prov_doc is None: continue plugin = prov_doc.add_hermes_plugin("process", plugin_name) - new_strategy_generation = prov_doc.add_activity(data={"prov:wasAssociatedWith": plugin.ref}) - new_strategies = prov_doc.add_entity( - data={"prov:wasAttributedTo": plugin.ref, "prov:wasGeneratedBy": new_strategy_generation.ref} - ) + new_strategy_generation = prov_doc.add_activity(data={ + "schema:description": "generate new merge strategies", + "prov:wasAssociatedWith": plugin.ref + }) + new_strategies = prov_doc.add_entity(data={ # TODO: record strategies + "schema:description": f"new merge strategies of plugin {plugin_name}", + "prov:wasAttributedTo": plugin.ref, + "prov:wasGeneratedBy": new_strategy_generation.ref + }) if merged_strategies is None: merged_strategies = new_strategies strategy_action = new_strategy_generation continue strategy_action = prov_doc.add_activity(data={ + "schema:description": "merging the new strategies into the others", "prov:used": [merged_strategies.ref, new_strategies.ref], "prov:wasInformedBy": [strategy_action.ref, new_strategy_generation.ref], "prov:wasAssociatedWith": process_command.ref }) - merged_strategies = prov_doc.add_entity(data={ + merged_strategies = prov_doc.add_entity(data={ # TODO: record strategies + "schema:description": "the merge strategies of multiple plugins merged together", "prov:wasDerivedFrom": [merged_strategies.ref, new_strategies.ref], "prov:wasGeneratedBy": strategy_action.ref, "prov:wasAttributedTo": process_command.ref @@ -147,10 +155,14 @@ def __call__(self, args: argparse.Namespace) -> None: )) ] new_action = prov_doc.add_activity(data={ # load of new data + "schema:description": f"loads the data from {harvester} plugin", "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref], "prov:used": stored_results }) new_data = prov_doc.add_entity(data={ # new data to be merged + "@type": "schema:CreativeWork", + "schema:description": f"data loaded from {harvester} plugin", + "schema:text": str(metadata.compact()), # TODO: maybe "prov:value" instead? "prov:wasAttributedTo": plugin.ref, "prov:wasGeneratedBy": new_action.ref, "prov:wasDerivedFrom": stored_results @@ -158,11 +170,12 @@ def __call__(self, args: argparse.Namespace) -> None: if merged_any: # One pass must have been completed already. new_action = prov_doc.add_activity(data={ + "schema:description": "merges the old data object with the new data", "prov:used": [last_data.ref, new_data.ref], "prov:wasInformedBy": [last_action.ref, new_action.ref], "prov:wasAssociatedWith": process_command.ref }) # initial merge action of the merge - merged_doc.prov_objects = [new_action, new_data, last_data] + merged_doc.prov_objects = [new_action, new_data, last_data] # set the starting objects of the merge self.log.info(f"### Merge data from {harvester} plugin") # merge data into the merge dict @@ -186,32 +199,55 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info("## Store processed metadata") # store processed data ctx.prepare_step("process") + begin_store_at_time = datetime.datetime.now().isoformat() with ctx["result"] as result_ctx: result_ctx["codemeta"] = merged_doc.compact() result_ctx["context"] = {"@context": merged_doc.full_context} result_ctx["expanded"] = merged_doc.ld_value + stored_at_time = datetime.datetime.now().isoformat() if prov_doc is not None: write = prov_doc.add_activity(data={ + "schema:description": "Writes the processed metadata into the HERMES cache.", "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref, plugin.ref], "prov:used": last_data.ref, - "prov:wasInformedBy": last_action.ref + "prov:wasInformedBy": last_action.ref, + "prov:startedAtTime": begin_store_at_time, + "prov:endedAtTime": stored_at_time }) # TODO: add more info prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The compacted version of the processed metadata.", + "schema:text": str(merged_doc.compact()), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "process" / "result" / "codemeta.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": last_data.ref, - "prov:wasAttributedTo": hermes_cache.ref + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": stored_at_time }) prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The context of the processed metadata.", + "schema:text": str({"@context": merged_doc.full_context}), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "process" / "result" / "context.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": last_data.ref, - "prov:wasAttributedTo": hermes_cache.ref + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": stored_at_time }) prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The expanded version of the processed metadata.", + "schema:text": str(merged_doc.ld_value), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "process" / "result" / "expanded.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": last_data.ref, - "prov:wasAttributedTo": hermes_cache.ref + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": stored_at_time }) with ctx["provenance"] as cache: diff --git a/src/hermes/model/merge/container.py b/src/hermes/model/merge/container.py index a286c7e8..470d3161 100644 --- a/src/hermes/model/merge/container.py +++ b/src/hermes/model/merge/container.py @@ -245,9 +245,10 @@ def __setitem__(self: Self, key: str, value: Union[JSON_LD_VALUE, BASIC_TYPE, TI create_new_merged_data = last_merged_data is self.prov_objects[2] elif self.prov_objects[0] is not None: merge_activity = self.prov_doc.add_activity(data={ - "schema:name": "merge", - "schema:description": "foo", - "prov:used": {"@list": [self.prov_objects[2].ref, str(self.path+[key])]}, + "schema:name": f"merge values at {str(self.path+[key])}", + "schema:description": f"inserting value in the second 'used' value at {str(self.path+[key])} into the " + "first 'used' value at the same point", + "prov:used": {"@list": [self.prov_objects[2].ref, self.prov_objects[1].ref]}, "prov:wasInformedBy": self.prov_objects[0].ref }) create_new_merged_data = True @@ -257,7 +258,13 @@ def __setitem__(self: Self, key: str, value: Union[JSON_LD_VALUE, BASIC_TYPE, TI return self.prov_objects[0] = merge_activity if create_new_merged_data: + outer_most_parent = self + while outer_most_parent.parent != None: + outer_most_parent = outer_most_parent.parent self.prov_objects[2] = self.prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": f"software metadata after merge of values at {str(self.path+[key])}", + "schema:text": str(outer_most_parent.compact()), # TODO: maybe "prov:value" instead? "prov:wasAttributedTo": self.prov_doc.get_hermes_command("process").ref, "prov:wasGeneratedBy": merge_activity.ref, "prov:wasDerivedFrom": {"@list": [self.prov_objects[1].ref, self.prov_objects[2].ref]} @@ -319,10 +326,15 @@ def _merge_item( raise MergeError(f"Can't merge, no strategy found for key '{key}'.") if self.prov_objects[0] is not None: merge_activity = self.prov_doc.add_activity(data={ - "schema:name": "merge", - "schema:description": "foo", + "schema:name": f"merge values at {str(self.path+[key])}", + "schema:description": f"merge value in the second 'used' value at {str(self.path+[key])} into the " + "first 'used' value at the same point using the third 'used' value", "prov:wasAssociatedWith": self.prov_doc.get_hermes_command("process").ref, - "prov:used": {"@list": [self.prov_objects[1].ref, self.prov_objects[2].ref, str(self.path+[key])]}, + "prov:used": {"@list": [ + self.prov_objects[2].ref, + self.prov_objects[1].ref, + f"{merger.merge.__module__}.{merger.merge.__qualname__}" + ]}, "prov:wasInformedBy": self.prov_objects[0].ref }) self.prov_objects[0] = merge_activity From 2813e02701647818acafa6b99ec3f8d0141b65ff Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Fri, 24 Jul 2026 12:32:26 +0200 Subject: [PATCH 12/41] add curate provenance recording --- src/hermes/commands/curate/base.py | 108 ++++++++++++++++++++++++++++ src/hermes/commands/process/base.py | 6 +- 2 files changed, 112 insertions(+), 2 deletions(-) diff --git a/src/hermes/commands/curate/base.py b/src/hermes/commands/curate/base.py index 51f2da08..c250fb9d 100644 --- a/src/hermes/commands/curate/base.py +++ b/src/hermes/commands/curate/base.py @@ -5,6 +5,8 @@ # SPDX-FileContributor: Michael Meinel import argparse +import datetime +from typing import Optional from pydantic import BaseModel @@ -13,6 +15,7 @@ from hermes.model import SoftwareMetadata from hermes.model.context_manager import HermesContext from hermes.model.error import HermesValidationError +from hermes.model.provenance.ld_prov import ld_prov_list class HermesCuratePlugin(HermesPlugin): @@ -35,6 +38,14 @@ class HermesCurateCommand(HermesCommand): settings_class = CurateSettings def __call__(self, args: argparse.Namespace) -> None: + self.log.info("# Load provenance data from process step") + prov_doc = self.load_prov_doc() + if prov_doc is not None: + curate_command = prov_doc.get_hermes_command("curate") + curate_base_plugin = prov_doc.get_hermes_base_plugin("curate") + process_command = prov_doc.get_hermes_command("process") + hermes_cache = prov_doc.get_hermes_cache() + self.log.info("# Metadata curation") plugin_name = self.settings.plugin @@ -54,6 +65,9 @@ def __call__(self, args: argparse.Namespace) -> None: raise HermesValidationError("The results of the process step are invalid.") from e ctx.finalize_step("process") + # save loaded metadata now, because it could be altered in curation + loaded_metadata_str = str(metadata.compact()) + self.log.info(f"## Load curation plugin {plugin_name}") # load plugin try: @@ -72,6 +86,100 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info("## Store curated data") # store metadata + begin_store_at_time = datetime.datetime.now().isoformat() curated_metadata.write_to_cache(ctx, "result") + stored_at_time = datetime.datetime.now().isoformat() + + if prov_doc is not None: + curate_plugin = prov_doc.add_hermes_plugin("curate", plugin_name) + store_action_of_process = prov_doc.shallow_search(lambda node: ( + "prov:wasAssociatedWith" in node and + node["prov:wasAssociatedWith"] == [process_command.ref, hermes_cache.ref] and + "prov:wasInformedBy" in node + ))[0] + stored_results_of_process = [res.ref for res in prov_doc.shallow_search(lambda node: ( + "prov:wasGeneratedBy" in node and node["prov:wasGeneratedBy"] == [store_action_of_process.ref] + ))] + load_action = prov_doc.add_activity(data={ + "schema:description": "loads the data from process step", + "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref], + "prov:used": stored_results_of_process + }) + loaded_data = prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "data loaded from process step", + "schema:text": loaded_metadata_str, # TODO: maybe "prov:value" instead? + "prov:wasAttributedTo": [curate_command.ref, curate_plugin.ref, hermes_cache.ref], + "prov:wasGeneratedBy": load_action.ref, + "prov:wasDerivedFrom": stored_results_of_process + }) + curated_data = prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "curated metadata", + "schema:text": str(curated_metadata.compact()), # TODO: maybe "prov:value" instead? + "prov:wasAttributedTo": [curate_plugin.ref, curate_base_plugin.ref, curate_command.ref], + "prov:wasInfluencedBy": curate_plugin.ref, + "prov:wasGeneratedBy": load_action.ref, + "prov:wasDerivedFrom": loaded_data.ref + }) + write = prov_doc.add_activity(data={ + "schema:description": "Writes the processed metadata into the HERMES cache.", + "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref, curate_plugin.ref], + "prov:used": curated_data.ref, + "prov:startedAtTime": begin_store_at_time, + "prov:endedAtTime": stored_at_time + }) + # TODO: add more info + prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The compacted version of the processed metadata.", + "schema:text": str(curated_metadata.compact()), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "curate" / "result" / "codemeta.json").absolute().as_uri(), + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": curated_data.ref, + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": stored_at_time + }) + prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The context of the processed metadata.", + "schema:text": str({"@context": curated_metadata.full_context}), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "curate" / "result" / "context.json").absolute().as_uri(), + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": curated_data.ref, + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": stored_at_time + }) + prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The expanded version of the processed metadata.", + "schema:text": str(curated_metadata.ld_value), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "curate" / "result" / "expanded.json").absolute().as_uri(), + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": curated_data.ref, + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": stored_at_time + }) + + with ctx["provenance"] as cache: + cache["result"] = prov_doc.ld_value ctx.finalize_step("curate") + + def load_prov_doc(self) -> Optional[ld_prov_list]: + ctx = HermesContext() + ctx.prepare_step("process") + with ctx["provenance"] as cache: + try: + return ld_prov_list.load_ld_prov_list(cache["result"]) + except Exception: + self.log.warning( + "The provenance data from the harvest step could not be loaded. " + "Processing will proceed without collecting provenance data.", + exc_info=1 + ) + finally: + ctx.finalize_step("process") diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index b7d83a79..95e23d34 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -163,7 +163,7 @@ def __call__(self, args: argparse.Namespace) -> None: "@type": "schema:CreativeWork", "schema:description": f"data loaded from {harvester} plugin", "schema:text": str(metadata.compact()), # TODO: maybe "prov:value" instead? - "prov:wasAttributedTo": plugin.ref, + "prov:wasAttributedTo": [process_command.ref, hermes_cache.ref], "prov:wasGeneratedBy": new_action.ref, "prov:wasDerivedFrom": stored_results }) @@ -209,7 +209,7 @@ def __call__(self, args: argparse.Namespace) -> None: if prov_doc is not None: write = prov_doc.add_activity(data={ "schema:description": "Writes the processed metadata into the HERMES cache.", - "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref, plugin.ref], + "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref], "prov:used": last_data.ref, "prov:wasInformedBy": last_action.ref, "prov:startedAtTime": begin_store_at_time, @@ -269,3 +269,5 @@ def load_prov_doc(self) -> Optional[ld_prov_list]: "Processing will proceed without collecting provenance data.", exc_info=1 ) + finally: + ctx.finalize_step("harvest") From 7eb72cde11efcdfb992b5240c33abf7a4124d8e4 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Fri, 24 Jul 2026 15:12:15 +0200 Subject: [PATCH 13/41] add recording of version of plugins --- src/hermes/commands/curate/base.py | 2 +- src/hermes/commands/harvest/base.py | 11 ++++++++--- src/hermes/commands/process/base.py | 2 +- src/hermes/model/provenance/ld_prov.py | 16 +++++++++++----- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/hermes/commands/curate/base.py b/src/hermes/commands/curate/base.py index c250fb9d..e5479fdd 100644 --- a/src/hermes/commands/curate/base.py +++ b/src/hermes/commands/curate/base.py @@ -91,7 +91,7 @@ def __call__(self, args: argparse.Namespace) -> None: stored_at_time = datetime.datetime.now().isoformat() if prov_doc is not None: - curate_plugin = prov_doc.add_hermes_plugin("curate", plugin_name) + curate_plugin = prov_doc.add_hermes_plugin("curate", plugin_name, plugin_func) store_action_of_process = prov_doc.shallow_search(lambda node: ( "prov:wasAssociatedWith" in node and node["prov:wasAssociatedWith"] == [process_command.ref, hermes_cache.ref] and diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index 56799e9c..9d902b78 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -37,20 +37,25 @@ def load(self, func, source, *args, **kwargs): elif isinstance(source, Path): source_metadata["schema:url"] = source.absolute().as_uri() elif isinstance(source, str): - source_metadata["schema:url"] = Path(source).absolute().as_uri() + try: + source_metadata["schema:url"] = Path(source).absolute().as_uri() + except Exception: + source_metadata["schema:url"] = source io_operation = { "schema:description": "Load operation called with (" f"{source_metadata['schema:url'] if 'schema:url' in source_metadata else str(source)}" f"{', ' + str(args) if args else ''}{', ' + str(kwargs) if kwargs else ''}).", "schema:name": f"{func.__module__}.{func.__qualname__}" } + io_operation["prov:startedAtTime"] = datetime.datetime.now().isoformat() result = func(source, *args, **kwargs) + io_operation["prov:endedAtTime"] = datetime.datetime.now().isoformat() loaded_metadata = {"schema:description": "the loaded data", "schema:text": str(result)} self.io_operations.append((source_metadata, io_operation, loaded_metadata)) return result def write(): - # TODO: Implement + # TODO: Is this needed? If yes, it needs to be implemented pass @@ -136,7 +141,7 @@ def __call__(self, args: argparse.Namespace) -> None: remove_harvest_plugin_from_prov_doc(prov_doc, plugin_name) - plugin = prov_doc.add_hermes_plugin("harvest", plugin_name) + plugin = prov_doc.add_hermes_plugin("harvest", plugin_name, plugin_func) plugin_io_operations = plugin_func.io_operations outputs = [] io_ops = [] diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index 95e23d34..d3a4d2ee 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -91,7 +91,7 @@ def __call__(self, args: argparse.Namespace) -> None: if prov_doc is None: continue - plugin = prov_doc.add_hermes_plugin("process", plugin_name) + plugin = prov_doc.add_hermes_plugin("process", plugin_name, plugin_func) new_strategy_generation = prov_doc.add_activity(data={ "schema:description": "generate new merge strategies", "prov:wasAssociatedWith": plugin.ref diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index 767d58ad..45f12078 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -4,6 +4,7 @@ # SPDX-FileContributor: Michael Fritzsche +from importlib.metadata import metadata from typing import Optional, Union from typing_extensions import Self @@ -121,14 +122,19 @@ def init_hermes_agents(self) -> None: "prov:actedOnBehalfOf": command.ref }) - def add_hermes_plugin(self, step, name) -> ld_dict: - # TODO: add version - node = self.add_agent(data={ + def add_hermes_plugin(self, step, name, plugin) -> ld_dict: + data = { "@id": ld_prov_list.HERMES_PLUGIN_ID_FORMAT.format(step=step, name=name), "@type": "schema:SoftwareApplication", - "schema:name": f"{utils.hermes_name} {step} plugin '{name}'", + "schema:name": f"{plugin.__module__}.{plugin.__class__.__qualname__}", + "schema:description": f"{utils.hermes_name} {step} plugin '{name}'", "prov:actedOnBehalfOf": self.get_hermes_base_plugin(step).ref - }) + } + try: + data["version"] = metadata(plugin.__module__)["version"] + except Exception: + pass + node = self.add_agent(data=data) return node def shallow_search(self, query) -> list[ld_dict]: From 483ea1ae4d124a76504a66515b3a3982dc4d4c59 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Fri, 31 Jul 2026 14:08:37 +0200 Subject: [PATCH 14/41] add recording of settings and args of commands --- src/hermes/commands/curate/base.py | 5 +- src/hermes/commands/harvest/base.py | 4 +- src/hermes/commands/process/base.py | 5 +- src/hermes/model/provenance/ld_prov.py | 90 +++++++++++++++++++++++++- 4 files changed, 99 insertions(+), 5 deletions(-) diff --git a/src/hermes/commands/curate/base.py b/src/hermes/commands/curate/base.py index e5479fdd..eb556e17 100644 --- a/src/hermes/commands/curate/base.py +++ b/src/hermes/commands/curate/base.py @@ -38,9 +38,12 @@ class HermesCurateCommand(HermesCommand): settings_class = CurateSettings def __call__(self, args: argparse.Namespace) -> None: + self.args = args self.log.info("# Load provenance data from process step") prov_doc = self.load_prov_doc() if prov_doc is not None: + prov_doc.add_hermes_settings(self) + prov_doc.add_settings_to_command("curate", self) curate_command = prov_doc.get_hermes_command("curate") curate_base_plugin = prov_doc.get_hermes_base_plugin("curate") process_command = prov_doc.get_hermes_command("process") @@ -91,7 +94,7 @@ def __call__(self, args: argparse.Namespace) -> None: stored_at_time = datetime.datetime.now().isoformat() if prov_doc is not None: - curate_plugin = prov_doc.add_hermes_plugin("curate", plugin_name, plugin_func) + curate_plugin = prov_doc.add_hermes_plugin("curate", plugin_name, plugin_func, self) store_action_of_process = prov_doc.shallow_search(lambda node: ( "prov:wasAssociatedWith" in node and node["prov:wasAssociatedWith"] == [process_command.ref, hermes_cache.ref] and diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index 9d902b78..680f24b3 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -102,6 +102,8 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info("# Load provenance from old harvest or create new document.") prov_doc = self.init_provenance_document() base_plugin = prov_doc.get_hermes_base_plugin("harvest") + prov_doc.add_hermes_settings(self) + prov_doc.add_settings_to_command("harvest", self) self.log.info("# Metadata harvesting") if len(self.settings.sources) == 0: @@ -141,7 +143,7 @@ def __call__(self, args: argparse.Namespace) -> None: remove_harvest_plugin_from_prov_doc(prov_doc, plugin_name) - plugin = prov_doc.add_hermes_plugin("harvest", plugin_name, plugin_func) + plugin = prov_doc.add_hermes_plugin("harvest", plugin_name, plugin_func, self) plugin_io_operations = plugin_func.io_operations outputs = [] io_ops = [] diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index d3a4d2ee..62741e28 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -41,9 +41,12 @@ class HermesProcessCommand(HermesCommand): settings_class = ProcessSettings def __call__(self, args: argparse.Namespace) -> None: + self.args = args self.log.info("# Load provenance data from harvest step") prov_doc = self.load_prov_doc() if prov_doc is not None: + prov_doc.add_hermes_settings(self) + prov_doc.add_settings_to_command("process", self) process_command = prov_doc.get_hermes_command("process") hermes_cache = prov_doc.get_hermes_cache() @@ -91,7 +94,7 @@ def __call__(self, args: argparse.Namespace) -> None: if prov_doc is None: continue - plugin = prov_doc.add_hermes_plugin("process", plugin_name, plugin_func) + plugin = prov_doc.add_hermes_plugin("process", plugin_name, plugin_func, self) new_strategy_generation = prov_doc.add_activity(data={ "schema:description": "generate new merge strategies", "prov:wasAssociatedWith": plugin.ref diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index 45f12078..3ed95a24 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -9,8 +9,9 @@ from typing_extensions import Self from hermes import utils +from hermes.commands.base import HermesCommand, HermesPlugin from hermes.model.types import ld_dict, ld_list -from hermes.model.types.ld_container import EXPANDED_JSON_LD_VALUE, JSON_LD_CONTEXT_DICT +from hermes.model.types.ld_container import BASIC_TYPE, EXPANDED_JSON_LD_VALUE, JSON_LD_CONTEXT_DICT from hermes.model.types.ld_context import ALL_CONTEXTS, iri_map @@ -122,14 +123,99 @@ def init_hermes_agents(self) -> None: "prov:actedOnBehalfOf": command.ref }) - def add_hermes_plugin(self, step, name, plugin) -> ld_dict: + def add_hermes_settings(self, command: HermesCommand) -> None: + hermes = self.get_hermes() + hermes.emplace("schema:supportingData") + hermes["schema:supportingData"].append({ + "@type": "schema:DataFeed", + "schema:dataFeedElement": [ + { + "@type": "schema:DataFeedItem", + "schema:name": name, + "schema:item": [ + { + "@type": "schema:Item", + "schema:description": value + } + ], + "schema:description": "setting provided by command line (or its default value)" + } + for name, value in [ + ("path", command.args.path.absolute().as_uri()), + ("config", command.args.config.absolute().as_uri()), + ("options", str(command.args.options)) + ] + ], + "schema:description": f"options for run {len(hermes['schema:supportingData']) + 1} of some hermes step" + }) + for name, values in command.root_settings.model_dump(mode="json").items(): + if not isinstance(values, list): + values = [values] + hermes["schema:supportingData"][-1]["schema:dataFeedElement"].append({ + "@type": "schema:DataFeedItem", + "schema:name": name, + "schema:item": [ + { + "@type": "schema:Item", + "schema:description": value + } + for value in values + ], + "schema:description": "setting loaded from the config file" + }) + + def add_settings_to_command(self, step: str, command: HermesCommand) -> None: + command_prov = self.get_hermes_command(step) + command_prov.emplace("schema:supportingData") + command_prov["schema:supportingData"].append({ + "@type": "schema:DataFeed", + "schema:dataFeedElement": [], + "schema:description": f"options for run {len(command_prov['schema:supportingData']) + 1} of step {step}" + }) + for name, values in command.settings.model_dump(mode="json").items(): + if not isinstance(values, list): + values = [values] + command_prov["schema:supportingData"][-1]["schema:dataFeedElement"].append({ + "@type": "schema:DataFeedItem", + "schema:name": name, + "schema:item": [ + { + "@type": "schema:Item", + "schema:description": value + } + for value in values + ] + }) + + def add_hermes_plugin(self, step: str, name: str, plugin: HermesPlugin, command: HermesCommand) -> ld_dict: data = { "@id": ld_prov_list.HERMES_PLUGIN_ID_FORMAT.format(step=step, name=name), "@type": "schema:SoftwareApplication", "schema:name": f"{plugin.__module__}.{plugin.__class__.__qualname__}", "schema:description": f"{utils.hermes_name} {step} plugin '{name}'", + "schema:supportingData": { + "@type": "schema:DataFeed", + "schema:dataFeedElement": [] + }, "prov:actedOnBehalfOf": self.get_hermes_base_plugin(step).ref } + try: + for name, values in getattr(command.settings, name).model_dump(mode="json").items(): + if not isinstance(values, list): + values = [values] + data["schema:supportingData"]["schema:dataFeedElement"].append({ + "@type": "schema:DataFeedItem", + "schema:name": name, + "schema:item": [ + { + "@type": "schema:Item", + "schema:description": value + } + for value in values + ] + }) + except Exception: + del data["schema:supportingData"] try: data["version"] = metadata(plugin.__module__)["version"] except Exception: From 22a9d7b105dfcd90183b515aaba007be4ec1401f Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Wed, 5 Aug 2026 13:03:06 +0200 Subject: [PATCH 15/41] add provenance recording for deposit --- src/hermes/commands/curate/base.py | 6 +- src/hermes/commands/deposit/base.py | 150 ++++++++++++++++++++++++- src/hermes/model/provenance/ld_prov.py | 5 +- 3 files changed, 150 insertions(+), 11 deletions(-) diff --git a/src/hermes/commands/curate/base.py b/src/hermes/commands/curate/base.py index eb556e17..4e056079 100644 --- a/src/hermes/commands/curate/base.py +++ b/src/hermes/commands/curate/base.py @@ -112,7 +112,7 @@ def __call__(self, args: argparse.Namespace) -> None: "@type": "schema:CreativeWork", "schema:description": "data loaded from process step", "schema:text": loaded_metadata_str, # TODO: maybe "prov:value" instead? - "prov:wasAttributedTo": [curate_command.ref, curate_plugin.ref, hermes_cache.ref], + "prov:wasAttributedTo": hermes_cache.ref, "prov:wasGeneratedBy": load_action.ref, "prov:wasDerivedFrom": stored_results_of_process }) @@ -127,7 +127,7 @@ def __call__(self, args: argparse.Namespace) -> None: }) write = prov_doc.add_activity(data={ "schema:description": "Writes the processed metadata into the HERMES cache.", - "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref, curate_plugin.ref], + "prov:wasAssociatedWith": [curate_command.ref, hermes_cache.ref], "prov:used": curated_data.ref, "prov:startedAtTime": begin_store_at_time, "prov:endedAtTime": stored_at_time @@ -180,7 +180,7 @@ def load_prov_doc(self) -> Optional[ld_prov_list]: return ld_prov_list.load_ld_prov_list(cache["result"]) except Exception: self.log.warning( - "The provenance data from the harvest step could not be loaded. " + "The provenance data from the process step could not be loaded. " "Processing will proceed without collecting provenance data.", exc_info=1 ) diff --git a/src/hermes/commands/deposit/base.py b/src/hermes/commands/deposit/base.py index 57bed627..ace1e614 100644 --- a/src/hermes/commands/deposit/base.py +++ b/src/hermes/commands/deposit/base.py @@ -7,6 +7,8 @@ import abc import argparse +import datetime +from typing import Optional from pydantic import BaseModel @@ -15,6 +17,7 @@ from hermes.model.context_manager import HermesContext from hermes.model import SoftwareMetadata from hermes.model.error import HermesValidationError +from hermes.model.provenance.ld_prov import ld_prov_list class BaseDepositPlugin(HermesPlugin): @@ -23,36 +26,143 @@ class BaseDepositPlugin(HermesPlugin): TODO: describe workflow... needs refactoring to be less stateful! """ - def __call__(self, command: HermesCommand) -> None: + def __call__(self, command: HermesCommand, prov_doc: ld_prov_list) -> None: """Initiate the deposition process. This calls a list of additional methods on the class, none of which need to be implemented. """ self.command = command + target = command.settings.target self.ctx = HermesContext() self.ctx.prepare_step("deposit") self.ctx.prepare_step("curate") try: + start_of_load = datetime.datetime.now().isoformat() self.metadata = SoftwareMetadata.load_from_cache(self.ctx, "result") + end_of_load = datetime.datetime.now().isoformat() except Exception as e: raise HermesValidationError("The results of the curate step are invalid.") from e self.ctx.finalize_step("curate") + if prov_doc is not None: + plugin = prov_doc.add_hermes_plugin("deposit", target, self, command) + deposit_command = prov_doc.get_hermes_command("deposit") + curate_command = prov_doc.get_hermes_command("curate") + deposit_base_plugin = prov_doc.get_hermes_base_plugin("deposit") + hermes_cache = prov_doc.get_hermes_cache() + store_action_curate = prov_doc.shallow_search(lambda node: ( + "prov:wasAssociatedWith" in node and + node["prov:wasAssociatedWith"] == [curate_command.ref, hermes_cache.ref] and + "prov:used" in node and + len(node["prov:used"]) == 1 + ))[0] + results_curate = [item.ref for item in prov_doc.shallow_search(lambda node: ( + "prov:wasGeneratedBy" in node and node["prov:wasGeneratedBy"] == [store_action_curate.ref] + ))] + load_action = prov_doc.add_activity(data={ + "schema:description": "Loads the results of the curate step.", + "prov:used": results_curate, + "prov:wasAssociatedWith": [hermes_cache.ref, deposit_command.ref, deposit_base_plugin.ref], + "prov:startedAtTime": start_of_load, + "prov:endedAtTime": end_of_load + }) + loaded_data = prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "data loaded from curate step", + "schema:text": str(self.metadata.compact()), # TODO: maybe "prov:value" instead? + "prov:wasAttributedTo": hermes_cache.ref, + "prov:wasGeneratedBy": load_action.ref, + "prov:wasDerivedFrom": results_curate, + "prov:generatedAtTime": end_of_load + }) + self.prepare() + start_of_map = datetime.datetime.now().isoformat() deposit = self.map_metadata() - with self.ctx[command.settings.target] as cache: + end_of_map = datetime.datetime.now().isoformat() + with self.ctx[target] as cache: + start_of_store = datetime.datetime.now().isoformat() cache["deposit"] = deposit + end_of_store = datetime.datetime.now().isoformat() + + if prov_doc is not None: + map_action = prov_doc.add_activity(data={ + "schema:description": "Maps the metadata to the format required by the deposition target.", + "prov:used": loaded_data.ref, + "prov:wasAssociatedWith": plugin.ref, + "prov:startedAtTime": start_of_map, + "prov:endedAtTime": end_of_map + }) + mapped_data = prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The metadata mapped to the format required by the deposition target.", + "schema:text": str(deposit), # TODO: maybe "prov:value" instead? + "prov:wasAttributedTo": plugin.ref, + "prov:wasGeneratedBy": map_action.ref, + "prov:wasDerivedFrom": loaded_data.ref, + "prov:generatedAtTime": end_of_load + }) + store_mapped_data = prov_doc.add_activity(data={ + "schema:description": "Stores the mapped metadata.", + "prov:used": mapped_data.ref, + "prov:wasAssociatedWith": [hermes_cache.ref, deposit_command.ref, deposit_base_plugin.ref], + "prov:startedAtTime": start_of_store, + "prov:endedAtTime": end_of_store + }) + prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The stored version of the mapped metadata.", + "schema:text": str(deposit), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (self.ctx.cache_dir / "deposit" / target / "deposit.json").absolute().as_uri(), + "prov:wasGeneratedBy": store_mapped_data.ref, + "prov:wasDerivedFrom": mapped_data.ref, + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": end_of_store + }) if self.is_initial_publication(): self.create_initial_version() else: self.create_new_version() - deposit = self.update_metadata() - with self.ctx[command.settings.target] as cache: - cache["result"] = deposit + updated_deposit = self.update_metadata() + end_of_update_map = datetime.datetime.now().isoformat() + with self.ctx[target] as cache: + start_of_second_store = datetime.datetime.now().isoformat() + cache["result"] = updated_deposit + end_of_second_store = datetime.datetime.now().isoformat() self.ctx.finalize_step("deposit") + + if prov_doc is not None: + updated_mapped_data = prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The updated mapped metadata.", + "schema:text": str(updated_deposit), # TODO: maybe "prov:value" instead? + "prov:wasInfluencedBy": plugin.ref, + "prov:wasDerivedFrom": mapped_data.ref, + "prov:generatedAtTime": end_of_update_map + }) + store_updated_mapped_data = prov_doc.add_activity(data={ + "schema:description": "Stores the mapped metadata.", + "prov:used": updated_mapped_data.ref, + "prov:wasAssociatedWith": [hermes_cache.ref, deposit_command.ref, deposit_base_plugin.ref], + "prov:startedAtTime": start_of_second_store, + "prov:endedAtTime": end_of_second_store + }) + prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The stored version of the updated mapped metadata.", + "schema:text": str(updated_deposit), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (self.ctx.cache_dir / "deposit" / target / "result.json").absolute().as_uri(), + "prov:wasGeneratedBy": store_updated_mapped_data.ref, + "prov:wasDerivedFrom": updated_mapped_data.ref, + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": end_of_second_store + }) + self.delete_artifacts() self.upload_artifacts() self.publish() @@ -138,6 +248,10 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info("# Metadata deposition") self.args = args plugin_name = self.settings.target + prov_doc = self.load_prov_doc() + if prov_doc is not None: + prov_doc.add_hermes_settings(self) + prov_doc.add_settings_to_command("deposit", self) self.log.info(f"## Load deposit plugin {plugin_name}") # load plugin @@ -150,9 +264,33 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info(f"## Run deposit plugin {plugin_name}") # run plugin try: - plugin_func(self) + plugin_func(self, prov_doc) except HermesValidationError as e: self.log.critical(f"## Error while executing {plugin_name} plugin.", exc_info=1) raise HermesPluginRunError( f"Something went wrong while running the deposit plugin {self.settings.plugin}" ) from e + + if prov_doc is None: + return + + ctx = HermesContext() + ctx.prepare_step("deposit") + with ctx["provenance"] as cache: + cache["result"] = prov_doc.ld_value + ctx.finalize_step("deposit") + + def load_prov_doc(self) -> Optional[ld_prov_list]: + ctx = HermesContext() + ctx.prepare_step("curate") + with ctx["provenance"] as cache: + try: + return ld_prov_list.load_ld_prov_list(cache["result"]) + except Exception: + self.log.warning( + "The provenance data from the curate step could not be loaded. " + "Deposition will proceed without collecting provenance data.", + exc_info=1 + ) + finally: + ctx.finalize_step("curate") diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index 3ed95a24..e9c1850a 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -170,8 +170,9 @@ def add_settings_to_command(self, step: str, command: HermesCommand) -> None: command_prov["schema:supportingData"].append({ "@type": "schema:DataFeed", "schema:dataFeedElement": [], - "schema:description": f"options for run {len(command_prov['schema:supportingData']) + 1} of step {step}" - }) + "schema:description": f"options for run {len(command_prov['schema:supportingData']) + 1} of step {step} out" + f" of {len(self.get_hermes()['schema:supportingData'])} runs of some hermes step" + }) # Needs add_hermes_settings to be called before add_settings_to_command is called! for name, values in command.settings.model_dump(mode="json").items(): if not isinstance(values, list): values = [values] From 7063755771842a126c0cc2a242c1b6de072add52 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Wed, 5 Aug 2026 15:37:31 +0200 Subject: [PATCH 16/41] revised provenance recording --- src/hermes/commands/curate/base.py | 13 ++++++++--- src/hermes/commands/harvest/base.py | 4 +++- src/hermes/commands/process/base.py | 32 +++++++++++++++++++++----- src/hermes/model/merge/container.py | 9 +++++++- src/hermes/model/provenance/ld_prov.py | 2 +- 5 files changed, 48 insertions(+), 12 deletions(-) diff --git a/src/hermes/commands/curate/base.py b/src/hermes/commands/curate/base.py index 4e056079..b8fc536b 100644 --- a/src/hermes/commands/curate/base.py +++ b/src/hermes/commands/curate/base.py @@ -59,7 +59,9 @@ def __call__(self, args: argparse.Namespace) -> None: # load processed data ctx.prepare_step("process") try: + begin_load_at_time = datetime.datetime.now().isoformat() metadata = SoftwareMetadata.load_from_cache(ctx, "result") + end_load_at_time = datetime.datetime.now().isoformat() except Exception as e: self.log.critical( "## The data from the process step could not be loaded or is invalid for some reason.", @@ -83,6 +85,7 @@ def __call__(self, args: argparse.Namespace) -> None: # run plugin try: curated_metadata = plugin_func(self, metadata) + end_curation_time = datetime.datetime.now().isoformat() except Exception as e: self.log.critical(f"## Unknown error while executing the {plugin_name} plugin.", exc_info=1) raise HermesPluginRunError(f"Something went wrong while running the curate plugin {plugin_name}") from e @@ -106,7 +109,9 @@ def __call__(self, args: argparse.Namespace) -> None: load_action = prov_doc.add_activity(data={ "schema:description": "loads the data from process step", "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref], - "prov:used": stored_results_of_process + "prov:used": stored_results_of_process, + "prov:startedAtTime": begin_load_at_time, + "prov:endedAtTime": end_load_at_time }) loaded_data = prov_doc.add_entity(data={ "@type": "schema:CreativeWork", @@ -114,7 +119,8 @@ def __call__(self, args: argparse.Namespace) -> None: "schema:text": loaded_metadata_str, # TODO: maybe "prov:value" instead? "prov:wasAttributedTo": hermes_cache.ref, "prov:wasGeneratedBy": load_action.ref, - "prov:wasDerivedFrom": stored_results_of_process + "prov:wasDerivedFrom": stored_results_of_process, + "prov:generatedAtTime": end_load_at_time }) curated_data = prov_doc.add_entity(data={ "@type": "schema:CreativeWork", @@ -123,7 +129,8 @@ def __call__(self, args: argparse.Namespace) -> None: "prov:wasAttributedTo": [curate_plugin.ref, curate_base_plugin.ref, curate_command.ref], "prov:wasInfluencedBy": curate_plugin.ref, "prov:wasGeneratedBy": load_action.ref, - "prov:wasDerivedFrom": loaded_data.ref + "prov:wasDerivedFrom": loaded_data.ref, + "prov:generatedAtTime": end_curation_time }) write = prov_doc.add_activity(data={ "schema:description": "Writes the processed metadata into the HERMES cache.", diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index 680f24b3..32550181 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -167,10 +167,12 @@ def __call__(self, args: argparse.Namespace) -> None: "prov:wasInformedBy": io_ops, "prov:used": outputs, "prov:wasAssociatedWith": plugin.ref, - "prov:startedAtTime": returned_at_time + "prov:endedAtTime": returned_at_time }) data_output = prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", "schema:description": "the harvested metadata", + "schema:text": str(harvested_data.compact()), # TODO: maybe "prov:value" instead? "prov:wasAttributedTo": plugin.ref, "prov:wasGeneratedBy": map_activity.ref, "prov:wasDerivedFrom": outputs, diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index 62741e28..eda4962c 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -82,14 +82,18 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info(f"### Run {plugin_name} plugin") # run plugin try: + generate_strategies_start = datetime.datetime.now().isoformat() additional_strategies = plugin_func(self) + generate_strategies_end = datetime.datetime.now().isoformat() except Exception: self.log.exception(f"### Unknown error while executing the {plugin_name} plugin, skipping it now.") continue self.log.info(f"### Add the strategies to the merge document {plugin_name} plugin") # add strategies to the merge document + merge_strategies_start = datetime.datetime.now().isoformat() merged_doc.add_strategy(additional_strategies) + merge_strategies_end = datetime.datetime.now().isoformat() any_strategies_loaded = True if prov_doc is None: @@ -97,12 +101,15 @@ def __call__(self, args: argparse.Namespace) -> None: plugin = prov_doc.add_hermes_plugin("process", plugin_name, plugin_func, self) new_strategy_generation = prov_doc.add_activity(data={ "schema:description": "generate new merge strategies", - "prov:wasAssociatedWith": plugin.ref + "prov:wasAssociatedWith": plugin.ref, + "prov:startedAtTime": generate_strategies_start, + "prov:endedAtTime": generate_strategies_end }) new_strategies = prov_doc.add_entity(data={ # TODO: record strategies "schema:description": f"new merge strategies of plugin {plugin_name}", "prov:wasAttributedTo": plugin.ref, - "prov:wasGeneratedBy": new_strategy_generation.ref + "prov:wasGeneratedBy": new_strategy_generation.ref, + "prov:generatedAtTime": generate_strategies_end }) if merged_strategies is None: merged_strategies = new_strategies @@ -112,13 +119,16 @@ def __call__(self, args: argparse.Namespace) -> None: "schema:description": "merging the new strategies into the others", "prov:used": [merged_strategies.ref, new_strategies.ref], "prov:wasInformedBy": [strategy_action.ref, new_strategy_generation.ref], - "prov:wasAssociatedWith": process_command.ref + "prov:wasAssociatedWith": process_command.ref, + "prov:startedAtTime": merge_strategies_start, + "prov:endedAtTime": merge_strategies_end }) merged_strategies = prov_doc.add_entity(data={ # TODO: record strategies "schema:description": "the merge strategies of multiple plugins merged together", "prov:wasDerivedFrom": [merged_strategies.ref, new_strategies.ref], "prov:wasGeneratedBy": strategy_action.ref, - "prov:wasAttributedTo": process_command.ref + "prov:wasAttributedTo": process_command.ref, + "prov:generatedAtTime": merge_strategies_end }) if not any_strategies_loaded: @@ -135,7 +145,9 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info(f"### Load data from {harvester} plugin") # load data from harvester try: + load_start = datetime.datetime.now().isoformat() metadata = SoftwareMetadata.load_from_cache(ctx, harvester) + load_end = datetime.datetime.now().isoformat() except Exception: # skip this harvester when the data is invalid if prov_doc is not None: @@ -160,7 +172,9 @@ def __call__(self, args: argparse.Namespace) -> None: new_action = prov_doc.add_activity(data={ # load of new data "schema:description": f"loads the data from {harvester} plugin", "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref], - "prov:used": stored_results + "prov:used": stored_results, + "prov:startedAtTime": load_start, + "prov:endedAtTime": load_end }) new_data = prov_doc.add_entity(data={ # new data to be merged "@type": "schema:CreativeWork", @@ -168,7 +182,8 @@ def __call__(self, args: argparse.Namespace) -> None: "schema:text": str(metadata.compact()), # TODO: maybe "prov:value" instead? "prov:wasAttributedTo": [process_command.ref, hermes_cache.ref], "prov:wasGeneratedBy": new_action.ref, - "prov:wasDerivedFrom": stored_results + "prov:wasDerivedFrom": stored_results, + "prov:generatedAtTime": load_end }) if merged_any: # One pass must have been completed already. @@ -183,13 +198,18 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info(f"### Merge data from {harvester} plugin") # merge data into the merge dict try: + merge_start = datetime.datetime.now().isoformat() merged_doc.update(metadata) + merge_end = datetime.datetime.now().isoformat() except Exception as e: # TODO: Maybe this state is recoverable by starting over again and skipping this plugin. self.log.critical(f"### Merging the data from {harvester} plugin resulted in an error.", exc_info=True) raise RuntimeError(f"Merging the data from {harvester} plugin failed.") from e if prov_doc is not None: + if merged_any: + new_action["prov:startedAtTime"] = merge_start + new_action["prov:endedAtTime"] = merge_end last_action = merged_doc.prov_objects[0] if merged_any else new_action last_data = merged_doc.prov_objects[2] if merged_any else new_data merged_any = True diff --git a/src/hermes/model/merge/container.py b/src/hermes/model/merge/container.py index 470d3161..c2330029 100644 --- a/src/hermes/model/merge/container.py +++ b/src/hermes/model/merge/container.py @@ -7,6 +7,7 @@ from __future__ import annotations +import datetime from typing import TYPE_CHECKING, Any, Callable, Optional, Union from typing_extensions import Self @@ -237,6 +238,7 @@ def __setitem__(self: Self, key: str, value: Union[JSON_LD_VALUE, BASIC_TYPE, TI ``self[key]``. """ # create the new item if self[key] and value have to be merged. + merge_start = datetime.datetime.now().isoformat() if key in self: if self.prov_objects[0] is not None: last_merged_data = self.prov_objects[2] @@ -254,8 +256,12 @@ def __setitem__(self: Self, key: str, value: Union[JSON_LD_VALUE, BASIC_TYPE, TI create_new_merged_data = True # update the entry of self[key] super().__setitem__(key, value) + merge_end = datetime.datetime.now().isoformat() if self.prov_objects[0] is None: return + if merge_activity is not None: + merge_activity["prov:startedAtTime"] = merge_start + merge_activity["prov:endedAtTime"] = merge_end self.prov_objects[0] = merge_activity if create_new_merged_data: outer_most_parent = self @@ -267,7 +273,8 @@ def __setitem__(self: Self, key: str, value: Union[JSON_LD_VALUE, BASIC_TYPE, TI "schema:text": str(outer_most_parent.compact()), # TODO: maybe "prov:value" instead? "prov:wasAttributedTo": self.prov_doc.get_hermes_command("process").ref, "prov:wasGeneratedBy": merge_activity.ref, - "prov:wasDerivedFrom": {"@list": [self.prov_objects[1].ref, self.prov_objects[2].ref]} + "prov:wasDerivedFrom": {"@list": [self.prov_objects[1].ref, self.prov_objects[2].ref]}, + "prov:generatedAtTime": merge_end }) else: self.prov_objects[2]["prov:wasGeneratedBy"].append(merge_activity.ref) diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index e9c1850a..ba86c274 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -218,7 +218,7 @@ def add_hermes_plugin(self, step: str, name: str, plugin: HermesPlugin, command: except Exception: del data["schema:supportingData"] try: - data["version"] = metadata(plugin.__module__)["version"] + data["schema:softwareVersion"] = metadata(plugin.__module__)["version"] except Exception: pass node = self.add_agent(data=data) From 1e83028a00a9b89390a42d34c13c316498710d92 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Wed, 5 Aug 2026 15:38:22 +0200 Subject: [PATCH 17/41] add diagrams of provenance for future use in adr (exists only on develop) --- .../hermes-prov-diagram/hermes-prov.drawio | 4162 +++++++++++++++++ .../hermes-prov.drawio.license | 3 + docs/adr/hermes-prov-diagram/hermes-prov.svg | 4 + .../hermes-prov.svg.license | 3 + 4 files changed, 4172 insertions(+) create mode 100644 docs/adr/hermes-prov-diagram/hermes-prov.drawio create mode 100644 docs/adr/hermes-prov-diagram/hermes-prov.drawio.license create mode 100644 docs/adr/hermes-prov-diagram/hermes-prov.svg create mode 100644 docs/adr/hermes-prov-diagram/hermes-prov.svg.license diff --git a/docs/adr/hermes-prov-diagram/hermes-prov.drawio b/docs/adr/hermes-prov-diagram/hermes-prov.drawio new file mode 100644 index 00000000..70b06617 --- /dev/null +++ b/docs/adr/hermes-prov-diagram/hermes-prov.drawio @@ -0,0 +1,4162 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/adr/hermes-prov-diagram/hermes-prov.drawio.license b/docs/adr/hermes-prov-diagram/hermes-prov.drawio.license new file mode 100644 index 00000000..e4d2c6e9 --- /dev/null +++ b/docs/adr/hermes-prov-diagram/hermes-prov.drawio.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2026 German Aerospace Center (DLR) + +SPDX-License-Identifier: CC-BY-SA-4.0 \ No newline at end of file diff --git a/docs/adr/hermes-prov-diagram/hermes-prov.svg b/docs/adr/hermes-prov-diagram/hermes-prov.svg new file mode 100644 index 00000000..56e2a032 --- /dev/null +++ b/docs/adr/hermes-prov-diagram/hermes-prov.svg @@ -0,0 +1,4 @@ + + + +
wasGeneratedBy
used
used
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
wasAssociatedWith
wasAttributedTo
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasInfluencedBy
used
wasDerivedFrom
actedOnBehalfOf
used
used
used
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
actedOnBehalfOf
wasAttributedTo
wasAssociatedWith
actedOnBehalfOf
wasDerivedFrom
wasInfluencedBy
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasAttributedTo
wasAttributedTo
used
used
wasAssociatedWith
wasGeneratedBy
wasAssociatedWith
wasAttributedTo
wasAssociatedWith
wasInformedBy
wasAttributedTo
wasGeneratedBy
wasInformedBy
wasGeneratedBy
used
used
wasGeneratedBy
wasInformedBy
used
wasDerivedFrom
wasDerivedFrom
actedOnBehalfOf
wasAttributedTo
wasAssociatedWith
used
used
used
used
used
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
used
used
wasAssociatedWith
wasInformedBy
used
used
wasInformedBy
wasGeneratedBy
wasInformedBy
used
wasDerivedFrom
wasGeneratedBy
used
wasInformedBy
wasDerivedFrom
wasGeneratedBy
wasDerivedFrom
used
wasGeneratedBy
wasInformedBy
used
wasGeneratedBy
wasGeneratedBy
used
used
used
used
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasInformedBy
used
used
used
used
used
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
used
wasAssociatedWith
wasInformedBy
used
wasInformedBy
wasGeneratedBy
wasInformedBy
used
wasDerivedFrom
wasGeneratedBy
used
wasInformedBy
wasDerivedFrom
wasGeneratedBy
wasDerivedFrom
used
wasGeneratedBy
wasInformedBy
used
wasGeneratedBy
wasGeneratedBy
used
used
used
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasAttributedTo
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasDerivedFrom
wasAssociatedWith
wasInformedBy
wasAttributedTo
wasGeneratedBy
used
wasDerivedFrom
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasAttributedTo
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
used
wasGeneratedBy
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
actedOnBehalfOf
wasGeneratedBy
wasGeneratedBy
actedOnBehalfOf
wasGeneratedBy
wasAttributedTo
wasAssociatedWith
actedOnBehalfOf
used
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasAttributedTo
used
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasAttributedTo
used
used
used
wasAssociatedWith
used
used
wasAssociatedWith
wasInformedBy
wasInformedBy
wasAttributedTo
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
used
used
used
wasAttributedTo
wasAssociatedWith
wasDerivedFrom
wasGeneratedBy
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
wasAssociatedWith
wasInformedBy
wasInformedBy
wasAssociatedWith
wasAttributedTo
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
used
used
used
wasAttributedTo
wasAssociatedWith
wasDerivedFrom
wasGeneratedBy
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasInformedBy
wasInformedBy
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
actedOnBehalfOf
actedOnBehalfOf
wasAttributedTo
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
used
used
used
wasAttributedTo
wasAssociatedWith
wasGeneratedBy
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
actedOnBehalfOf
harvest plugin

name, version, settings
harvest source

path uri
.hermes/harvest/
{plugin_name}/codemeta.json

text, path uri, date created
harvested metadata

data
software-metadata

date, time
map

end time
write

start time, end time
.hermes/harvest/
{plugin_name}/expanded.json

text, path uri, date created
.hermes/harvest/
{plugin_name}/context.json

text, path uri, date created
Legend
design
meaning
provenance: Agent
provenance: Entity
provenance: Activity
bold text
record those properties always
solid lining
record as detailed as possible
dashed lining
record without many details
grayed out
optional / not always existent
name
properties
name
properties
name
properties
harvest plugin

name, version, settings
harvest source

path uri
.hermes/harvest/
{plugin_name}/codemeta.json

text, path uri, date created
harvested metadata

data
software-metadata

data, time
map

end time
write

start time, end time
.hermes/harvest/
{plugin_name}/expanded.json

text, path uri, date created
HARVEST
hermes

version
HERMES cache
load

func, args, kwargs, source, time
harvest base plugin

settings
load

func, args, kwargs, source, time
harvest command

settings
.hermes/harvest/
{plugin_name}/context.json

text, path uri, date created
process plugin

name, version, settings
merge strategies

strategies, time
process plugin

name, version, settings
.hermes/process/result/
codemeta.json

text, path uri, date created
merge strategies

strategies, time
merge strategies

start time, end time
write

start time, end time
.hermes/process/result/
expanded.json

text, path uri, date created
PROCESS
generate merge strategies

start time, end time
process base plugin

settings
generate merge strategies

start time, end time
process command

settings
.hermes/process/result/
context.json

text, path uri, date created
process plugin

name, version, settings
generate merge strategies

start time, end time
merge strategies

start time, end time
merged strategies

strategies, time
merged strategies

strategies, time
harvest plugin

name, version, settings
harvest source

path uri
.hermes/harvest/
{plugin_name}/codemeta.json

text, path uri, date created
harvested metadata

data
software-metadata

data, time
map

end time
write

start time, end time
.hermes/harvest/
{plugin_name}/expanded.json

text, path uri, date created
.hermes/harvest/
{plugin_name}/context.json

text, path uri, date created
load

func, args, kwargs, source, time
software-metadata

data, time
load

start time, end time
software-metadata

data, time
load

start time, end time
software-metadata

data, time
load

start time, end time
used
used
wasAssociatedWith
reject/ replace/ ...
value with other value

start time, end time, strategy used
merge value at key

key, strategy used
merge

start time, end time
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
software-metadata

time, data
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
reject/ replace/ ...
value with other value

start time, end time, strategy used
merge value at key

key, strategy used
merge

start time, end time
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
software-metadata

time, data
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
merge strategies

strategies, time
curate command

settings
software-metadata

data, time
load

start time, end time
curate base plugin

settings
curate plugin

name, version, settings
software-metadata

data, time
.hermes/curate/result/
codemeta.json

text, path uri, time created
write

start time, end time
.hermes/curate/result/
expanded.json

text, path uri, time created
.hermes/curate/result/
context.json

text, path uri, time created
CURATE
used
wasDerivedFrom
actedOnBehalfOf
used
used
used
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
actedOnBehalfOf
wasAttributedTo
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
wasDerivedFrom
wasAttributedTo
wasAttributedTo
wasDerivedFrom
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
deposit command

settings
software-metadata

data, time
load

start time, end time
deposit base plugin

settings
deposit plugin

name, version, settings
mapped data for deposit

data, time
.hermes/deposit/
{deposit_plugin}/deposit.json

text, path uri, time created
write

start time, end time
DEPOSIT
map

start time, end time
updated metadata

data, time
.hermes/deposit/
{deposit_plugin}/result.json

text, path uri, time created
write

start time, end time
 \ No newline at end of file diff --git a/docs/adr/hermes-prov-diagram/hermes-prov.svg.license b/docs/adr/hermes-prov-diagram/hermes-prov.svg.license new file mode 100644 index 00000000..e4d2c6e9 --- /dev/null +++ b/docs/adr/hermes-prov-diagram/hermes-prov.svg.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2026 German Aerospace Center (DLR) + +SPDX-License-Identifier: CC-BY-SA-4.0 \ No newline at end of file From 66c61b2728523ceacd36ad6c12bb51d4f7c802e3 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Thu, 6 Aug 2026 11:55:48 +0200 Subject: [PATCH 18/41] add recording of merge strategies --- src/hermes/commands/process/base.py | 3 +++ src/hermes/model/merge/action.py | 18 ++++++++++++++++++ src/hermes/model/merge/container.py | 18 ++++++++++++------ 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index eda4962c..1fcba96e 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -106,7 +106,9 @@ def __call__(self, args: argparse.Namespace) -> None: "prov:endedAtTime": generate_strategies_end }) new_strategies = prov_doc.add_entity(data={ # TODO: record strategies + "@type": "schema:CreativeWork", "schema:description": f"new merge strategies of plugin {plugin_name}", + "schema:text": str(additional_strategies), # TODO: maybe "prov:value" instead? "prov:wasAttributedTo": plugin.ref, "prov:wasGeneratedBy": new_strategy_generation.ref, "prov:generatedAtTime": generate_strategies_end @@ -125,6 +127,7 @@ def __call__(self, args: argparse.Namespace) -> None: }) merged_strategies = prov_doc.add_entity(data={ # TODO: record strategies "schema:description": "the merge strategies of multiple plugins merged together", + "schema:text": str(merged_doc.strategies), # TODO: maybe "prov:value" instead? "prov:wasDerivedFrom": [merged_strategies.ref, new_strategies.ref], "prov:wasGeneratedBy": strategy_action.ref, "prov:wasAttributedTo": process_command.ref, diff --git a/src/hermes/model/merge/action.py b/src/hermes/model/merge/action.py index f2cfc7b3..9c52115c 100644 --- a/src/hermes/model/merge/action.py +++ b/src/hermes/model/merge/action.py @@ -49,6 +49,16 @@ def merge( """ raise NotImplementedError() + def __repr__(self) -> str: + """ + A generic stringify method for MergeActions. + Please overwrite this method if your MergeAction should be represented differently in the provenance data. + (I.e. if not all important attributes are recorded or some attributes string representation is not adequat.) + """ + if self.__dict__: + return f"{self.__module__}.{self.__class__.__qualname__} with attributes {str(self.__dict__)}" + return f"{self.__module__}.{self.__class__.__qualname__}" + class Reject(MergeAction): """ :class:`MergeAction` providing a merge function for rejecting the incoming item. """ @@ -209,6 +219,10 @@ def merge( return value + def __repr__(self): + return f"{self.__module__}.{self.__class__.__qualname__} with attributes " \ + f"{{'match': {self.match.__module__}.{self.match.__qualname__}, 'reject_incoming': {self.reject_incoming}}}" + class MergeSet(MergeAction): """ @@ -275,6 +289,10 @@ def merge( # Return the merged values. return value + def __repr__(self): + return f"{self.__module__}.{self.__class__.__qualname__} with attributes " \ + f"{{'match': {self.match.__module__}.{self.match.__qualname__}}}" + class IdMerge(MergeAction): """ :class:`MergeAction` providing a merge function for merging ids, i.e. error if not equals else do nothing. """ diff --git a/src/hermes/model/merge/container.py b/src/hermes/model/merge/container.py index c2330029..86c5a531 100644 --- a/src/hermes/model/merge/container.py +++ b/src/hermes/model/merge/container.py @@ -248,8 +248,8 @@ def __setitem__(self: Self, key: str, value: Union[JSON_LD_VALUE, BASIC_TYPE, TI elif self.prov_objects[0] is not None: merge_activity = self.prov_doc.add_activity(data={ "schema:name": f"merge values at {str(self.path+[key])}", - "schema:description": f"inserting value in the second 'used' value at {str(self.path+[key])} into the " - "first 'used' value at the same point", + "schema:description": f"Inserting value in the second 'used' value at {str(self.path+[key])} into the " + "first 'used' value at the same point, no merger needed.", "prov:used": {"@list": [self.prov_objects[2].ref, self.prov_objects[1].ref]}, "prov:wasInformedBy": self.prov_objects[0].ref }) @@ -324,8 +324,14 @@ def _merge_item( # search for all applicable strategies strategy = {**self.strategies.get(None, {})} ld_types = self.data_dict.get('@type', []) + type_of_used_strategy = None + key_of_used_strategy = None for ld_type in ld_types: strategy.update(self.strategies.get(ld_type, {})) + if key in self.strategies.get(ld_type, {}): + type_of_used_strategy = ld_type + key_of_used_strategy = key + # choose one merge strategy and return the item returned by following the merge startegy merger = strategy.get(key, strategy.get(None, None)) @@ -334,13 +340,13 @@ def _merge_item( if self.prov_objects[0] is not None: merge_activity = self.prov_doc.add_activity(data={ "schema:name": f"merge values at {str(self.path+[key])}", - "schema:description": f"merge value in the second 'used' value at {str(self.path+[key])} into the " - "first 'used' value at the same point using the third 'used' value", + "schema:description": f"Merge value in the second 'used' value at {str(self.path+[key])} into the " + f"first 'used' value at the same point using the merger {merger} for type " + f"{type_of_used_strategy} and key {key_of_used_strategy}", "prov:wasAssociatedWith": self.prov_doc.get_hermes_command("process").ref, "prov:used": {"@list": [ self.prov_objects[2].ref, - self.prov_objects[1].ref, - f"{merger.merge.__module__}.{merger.merge.__qualname__}" + self.prov_objects[1].ref ]}, "prov:wasInformedBy": self.prov_objects[0].ref }) From b775072a040bf0be11c1a1502c1b459551e4deae Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Thu, 6 Aug 2026 17:03:07 +0200 Subject: [PATCH 19/41] add merge strategies for invenio publish --- hermes.toml | 3 + pyproject.toml | 1 + src/hermes/commands/process/invenio_merge.py | 93 ++++++++++++++++++++ 3 files changed, 97 insertions(+) create mode 100644 src/hermes/commands/process/invenio_merge.py diff --git a/hermes.toml b/hermes.toml index a42a9406..dab72523 100644 --- a/hermes.toml +++ b/hermes.toml @@ -5,6 +5,9 @@ [harvest] sources = [ "cff", "toml" ] # ordered priority (first one is most important) +[process] +plugins = [ "invenio", "codemeta" ] + [curate] plugin = "pass_curate" diff --git a/pyproject.toml b/pyproject.toml index 17ea5087..8bb98ac6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,7 @@ codemeta_doi = "hermes.commands.postprocess.invenio:codemeta_doi" [project.entry-points."hermes.process"] codemeta = "hermes.commands.process.standard_merge:CodemetaProcessPlugin" +invenio = "hermes.commands.process.invenio_merge:InvenioProcessPlugin" [project.entry-points."hermes.curate"] pass_curate = "hermes.commands.curate.pass_curate:DoNothingCuratePlugin" diff --git a/src/hermes/commands/process/invenio_merge.py b/src/hermes/commands/process/invenio_merge.py new file mode 100644 index 00000000..39e5c6ec --- /dev/null +++ b/src/hermes/commands/process/invenio_merge.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: 2026 German Aerospace Center (DLR) +# +# SPDX-License-Identifier: Apache-2.0 + +# SPDX-FileContributor: Michael Fritzsche + + +from typing import Union +from typing_extensions import Self + +from hermes.commands.base import HermesCommand +from hermes.model.merge.action import MergeAction +from hermes.model.merge.container import ld_merge_dict, ld_merge_list +from hermes.model.types import ld_dict, ld_list +from hermes.model.types.ld_container import BASIC_TYPE, TIME_TYPE +from hermes.model.types.ld_context import iri_map as iri +from .base import HermesProcessPlugin + + +class InvenioMerge(MergeAction): + """ :class:`MergeAction` providing a merge function that tries to conform with Invenios metadata restrictions. """ + def merge( + self: Self, + target: ld_merge_dict, + key: list[Union[str, int]], + value: Union[ld_merge_list, str], + update: Union[BASIC_TYPE, TIME_TYPE, ld_dict, ld_list] + ) -> ld_merge_list: + print(key, value, update) + types = target.get("@type", []) + print(types) + if key[-1] == iri["schema:license"] and (iri["schema:SoftwareSourceCode"] in types or iri["schema:SoftwareApplication"] in types): + if len(value) == 1: + if isinstance(value[0], str) or ( + isinstance(value[0], (dict, ld_merge_dict)) and [*value[0].keys()] == ["@id"] + ): + if value != update: + target.reject(key, update) + return value + if isinstance(update, ld_list) and len(update) == 1: + if isinstance(update[0], str) or ( + isinstance(update[0], (dict, ld_merge_dict)) and [*update[0].keys()] == ["@id"] + ): + target.replace(key, value) + return update + target.reject(key, update) + return value + if ((key[-1] == iri["schema:familyName"] and iri["schema:Person"] in types) or + (key[-1] == iri["schema:name"] and iri["schema:Person"] in types) or + (key[-1] == iri["schema:name"] and (iri["schema:SoftwareSourceCode"] in types or iri["schema:SoftwareApplication"] in types)) + ): + if len(value) == 1: + if value != update: + target.reject(key, update) + return value + if len(update) == 1: + target.replace(key, value) + return update + if len(value) == len(update) == 0: + return value + target.reject(key, update) + return value + if ((key[-1] == iri["schema:version"] or key[-1] == iri["schema:description"]) and + (iri["schema:SoftwareSourceCode"] in types or iri["schema:SoftwareApplication"] in types) + ): + if len(value) == 1: + if value != update: + target.reject(key, update) + return value + if len(update) == 1: + target.replace(key, value) + return update + if len(value) == 0 or len(update) == 0: + return [] + target.reject(key, update) + return value + print("fail") + + +class InvenioProcessPlugin(HermesProcessPlugin): + def __call__(self, command: HermesCommand) -> dict[Union[str, None], dict[Union[str, None], MergeAction]]: + merger = InvenioMerge() + return { + iri["schema:SoftwareSourceCode"]: { + iri["schema:"+term]: merger for term in ["version", "name", "description", "license"] + }, + iri["schema:SoftwareApplication"]: { + iri["schema:"+term]: merger for term in ["version", "name", "description", "license"] + }, + iri["schema:Person"]: { + iri["schema:"+term]: merger for term in ["familyName", "name"] + } + } From 0cb53579a315145fc0e08f160f38649fff84245a Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Thu, 6 Aug 2026 17:11:06 +0200 Subject: [PATCH 20/41] fix issue 434 --- src/hermes/commands/curate/base.py | 10 +++++----- src/hermes/commands/deposit/base.py | 18 +++++++++--------- src/hermes/commands/harvest/base.py | 13 +++++++------ src/hermes/commands/process/base.py | 20 ++++++++++---------- src/hermes/model/merge/container.py | 4 ++-- src/hermes/model/types/ld_container.py | 8 ++++++++ 6 files changed, 41 insertions(+), 32 deletions(-) diff --git a/src/hermes/commands/curate/base.py b/src/hermes/commands/curate/base.py index b8fc536b..b151e9f0 100644 --- a/src/hermes/commands/curate/base.py +++ b/src/hermes/commands/curate/base.py @@ -59,9 +59,9 @@ def __call__(self, args: argparse.Namespace) -> None: # load processed data ctx.prepare_step("process") try: - begin_load_at_time = datetime.datetime.now().isoformat() + begin_load_at_time = datetime.datetime.now() metadata = SoftwareMetadata.load_from_cache(ctx, "result") - end_load_at_time = datetime.datetime.now().isoformat() + end_load_at_time = datetime.datetime.now() except Exception as e: self.log.critical( "## The data from the process step could not be loaded or is invalid for some reason.", @@ -85,16 +85,16 @@ def __call__(self, args: argparse.Namespace) -> None: # run plugin try: curated_metadata = plugin_func(self, metadata) - end_curation_time = datetime.datetime.now().isoformat() + end_curation_time = datetime.datetime.now() except Exception as e: self.log.critical(f"## Unknown error while executing the {plugin_name} plugin.", exc_info=1) raise HermesPluginRunError(f"Something went wrong while running the curate plugin {plugin_name}") from e self.log.info("## Store curated data") # store metadata - begin_store_at_time = datetime.datetime.now().isoformat() + begin_store_at_time = datetime.datetime.now() curated_metadata.write_to_cache(ctx, "result") - stored_at_time = datetime.datetime.now().isoformat() + stored_at_time = datetime.datetime.now() if prov_doc is not None: curate_plugin = prov_doc.add_hermes_plugin("curate", plugin_name, plugin_func, self) diff --git a/src/hermes/commands/deposit/base.py b/src/hermes/commands/deposit/base.py index ace1e614..b8beb1c5 100644 --- a/src/hermes/commands/deposit/base.py +++ b/src/hermes/commands/deposit/base.py @@ -38,9 +38,9 @@ def __call__(self, command: HermesCommand, prov_doc: ld_prov_list) -> None: self.ctx.prepare_step("curate") try: - start_of_load = datetime.datetime.now().isoformat() + start_of_load = datetime.datetime.now() self.metadata = SoftwareMetadata.load_from_cache(self.ctx, "result") - end_of_load = datetime.datetime.now().isoformat() + end_of_load = datetime.datetime.now() except Exception as e: raise HermesValidationError("The results of the curate step are invalid.") from e self.ctx.finalize_step("curate") @@ -78,13 +78,13 @@ def __call__(self, command: HermesCommand, prov_doc: ld_prov_list) -> None: }) self.prepare() - start_of_map = datetime.datetime.now().isoformat() + start_of_map = datetime.datetime.now() deposit = self.map_metadata() - end_of_map = datetime.datetime.now().isoformat() + end_of_map = datetime.datetime.now() with self.ctx[target] as cache: - start_of_store = datetime.datetime.now().isoformat() + start_of_store = datetime.datetime.now() cache["deposit"] = deposit - end_of_store = datetime.datetime.now().isoformat() + end_of_store = datetime.datetime.now() if prov_doc is not None: map_action = prov_doc.add_activity(data={ @@ -128,11 +128,11 @@ def __call__(self, command: HermesCommand, prov_doc: ld_prov_list) -> None: self.create_new_version() updated_deposit = self.update_metadata() - end_of_update_map = datetime.datetime.now().isoformat() + end_of_update_map = datetime.datetime.now() with self.ctx[target] as cache: - start_of_second_store = datetime.datetime.now().isoformat() + start_of_second_store = datetime.datetime.now() cache["result"] = updated_deposit - end_of_second_store = datetime.datetime.now().isoformat() + end_of_second_store = datetime.datetime.now() self.ctx.finalize_step("deposit") if prov_doc is not None: diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index 32550181..ec02f948 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -47,9 +47,9 @@ def load(self, func, source, *args, **kwargs): f"{', ' + str(args) if args else ''}{', ' + str(kwargs) if kwargs else ''}).", "schema:name": f"{func.__module__}.{func.__qualname__}" } - io_operation["prov:startedAtTime"] = datetime.datetime.now().isoformat() + io_operation["prov:startedAtTime"] = datetime.datetime.now() result = func(source, *args, **kwargs) - io_operation["prov:endedAtTime"] = datetime.datetime.now().isoformat() + io_operation["prov:endedAtTime"] = datetime.datetime.now() loaded_metadata = {"schema:description": "the loaded data", "schema:text": str(result)} self.io_operations.append((source_metadata, io_operation, loaded_metadata)) return result @@ -132,13 +132,13 @@ def __call__(self, args: argparse.Namespace) -> None: except Exception: self.log.exception(f"### Unknown error while executing the {plugin_name} plugin, skipping it now.") continue - returned_at_time = datetime.datetime.now().isoformat() + returned_at_time = datetime.datetime.now() self.log.info(f"### Store metadata harvested by {plugin_name} plugin") # store harvested data - begin_store_at_time = datetime.datetime.now().isoformat() + begin_store_at_time = datetime.datetime.now() harvested_data.write_to_cache(ctx, plugin_name) - stored_at_time = datetime.datetime.now().isoformat() + stored_at_time = datetime.datetime.now() harvested_any = True remove_harvest_plugin_from_prov_doc(prov_doc, plugin_name) @@ -233,7 +233,8 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.critical("No harvest plugin ran successfully.") raise HermesPluginRunError("No harvest plugin ran successfully.") - def init_provenance_document(self) -> ld_prov_list: + @classmethod + def init_provenance_document(cls) -> ld_prov_list: ctx = HermesContext() ctx.prepare_step("harvest") with ctx["provenance"] as cache: diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index 1fcba96e..8bea1b6b 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -82,18 +82,18 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info(f"### Run {plugin_name} plugin") # run plugin try: - generate_strategies_start = datetime.datetime.now().isoformat() + generate_strategies_start = datetime.datetime.now() additional_strategies = plugin_func(self) - generate_strategies_end = datetime.datetime.now().isoformat() + generate_strategies_end = datetime.datetime.now() except Exception: self.log.exception(f"### Unknown error while executing the {plugin_name} plugin, skipping it now.") continue self.log.info(f"### Add the strategies to the merge document {plugin_name} plugin") # add strategies to the merge document - merge_strategies_start = datetime.datetime.now().isoformat() + merge_strategies_start = datetime.datetime.now() merged_doc.add_strategy(additional_strategies) - merge_strategies_end = datetime.datetime.now().isoformat() + merge_strategies_end = datetime.datetime.now() any_strategies_loaded = True if prov_doc is None: @@ -148,9 +148,9 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info(f"### Load data from {harvester} plugin") # load data from harvester try: - load_start = datetime.datetime.now().isoformat() + load_start = datetime.datetime.now() metadata = SoftwareMetadata.load_from_cache(ctx, harvester) - load_end = datetime.datetime.now().isoformat() + load_end = datetime.datetime.now() except Exception: # skip this harvester when the data is invalid if prov_doc is not None: @@ -201,9 +201,9 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info(f"### Merge data from {harvester} plugin") # merge data into the merge dict try: - merge_start = datetime.datetime.now().isoformat() + merge_start = datetime.datetime.now() merged_doc.update(metadata) - merge_end = datetime.datetime.now().isoformat() + merge_end = datetime.datetime.now() except Exception as e: # TODO: Maybe this state is recoverable by starting over again and skipping this plugin. self.log.critical(f"### Merging the data from {harvester} plugin resulted in an error.", exc_info=True) @@ -225,12 +225,12 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info("## Store processed metadata") # store processed data ctx.prepare_step("process") - begin_store_at_time = datetime.datetime.now().isoformat() + begin_store_at_time = datetime.datetime.now() with ctx["result"] as result_ctx: result_ctx["codemeta"] = merged_doc.compact() result_ctx["context"] = {"@context": merged_doc.full_context} result_ctx["expanded"] = merged_doc.ld_value - stored_at_time = datetime.datetime.now().isoformat() + stored_at_time = datetime.datetime.now() if prov_doc is not None: write = prov_doc.add_activity(data={ diff --git a/src/hermes/model/merge/container.py b/src/hermes/model/merge/container.py index 86c5a531..fc77fdcc 100644 --- a/src/hermes/model/merge/container.py +++ b/src/hermes/model/merge/container.py @@ -238,7 +238,7 @@ def __setitem__(self: Self, key: str, value: Union[JSON_LD_VALUE, BASIC_TYPE, TI ``self[key]``. """ # create the new item if self[key] and value have to be merged. - merge_start = datetime.datetime.now().isoformat() + merge_start = datetime.datetime.now() if key in self: if self.prov_objects[0] is not None: last_merged_data = self.prov_objects[2] @@ -256,7 +256,7 @@ def __setitem__(self: Self, key: str, value: Union[JSON_LD_VALUE, BASIC_TYPE, TI create_new_merged_data = True # update the entry of self[key] super().__setitem__(key, value) - merge_end = datetime.datetime.now().isoformat() + merge_end = datetime.datetime.now() if self.prov_objects[0] is None: return if merge_activity is not None: diff --git a/src/hermes/model/types/ld_container.py b/src/hermes/model/types/ld_container.py index 10d2a82c..19fd70ae 100644 --- a/src/hermes/model/types/ld_container.py +++ b/src/hermes/model/types/ld_container.py @@ -13,6 +13,8 @@ from typing import Any, Optional, TypeAlias, TYPE_CHECKING, Union from typing_extensions import Self +from hermes.model.types.ld_context import iri_map + from .pyld_util import JsonLdProcessor, bundled_loader if TYPE_CHECKING: from .ld_dict import ld_dict @@ -462,6 +464,12 @@ def typed_ld_to_py(cls: type[Self], data: list[dict[str, BASIC_TYPE]], **kwargs) """ # FIXME: #434 dates are not returned as datetime/ date/ time but as string ld_value = data[0]['@value'] + if iri_map["schema:DateTime"] == data[0]['@type']: + ld_value = datetime.fromisoformat(ld_value) + elif iri_map["schema:Date"] == data[0]['@type']: + ld_value = date.fromisoformat(ld_value) + elif iri_map["schema:Time"] == data[0]['@type']: + ld_value = time.fromisoformat(ld_value) return ld_value From c3c78af0ee33dc3cdece4e8b301bc7f086a853c1 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Wed, 19 Aug 2026 17:16:15 +0200 Subject: [PATCH 21/41] added hermes report command, fixes issue 484 --- src/hermes/commands/__init__.py | 1 + src/hermes/commands/cli.py | 3 +- src/hermes/commands/process/base.py | 7 +- src/hermes/commands/process/invenio_merge.py | 3 - src/hermes/commands/report/__init__.py | 0 src/hermes/commands/report/base.py | 313 +++++++++++++++++++ src/hermes/model/merge/container.py | 13 +- 7 files changed, 329 insertions(+), 11 deletions(-) create mode 100644 src/hermes/commands/report/__init__.py create mode 100644 src/hermes/commands/report/base.py diff --git a/src/hermes/commands/__init__.py b/src/hermes/commands/__init__.py index 5203ac18..2733d02e 100644 --- a/src/hermes/commands/__init__.py +++ b/src/hermes/commands/__init__.py @@ -17,3 +17,4 @@ from hermes.commands.process.base import HermesProcessCommand from hermes.commands.deposit.base import HermesDepositCommand from hermes.commands.postprocess.base import HermesPostprocessCommand +from hermes.commands.report.base import HermesReportCommand diff --git a/src/hermes/commands/cli.py b/src/hermes/commands/cli.py index 68cc23e1..cbc645e0 100644 --- a/src/hermes/commands/cli.py +++ b/src/hermes/commands/cli.py @@ -14,7 +14,7 @@ from hermes import logger from hermes.commands import ( HermesCurateCommand, HermesCleanCommand, HermesDepositCommand, HermesHarvestCommand, HermesHelpCommand, - HermesInitCommand, HermesPostprocessCommand, HermesProcessCommand, HermesVersionCommand + HermesInitCommand, HermesPostprocessCommand, HermesProcessCommand, HermesReportCommand, HermesVersionCommand ) from hermes.commands.base import HermesCommand from hermes.error import HermesPluginRunError @@ -46,6 +46,7 @@ def main() -> None: HermesInitCommand(parser), HermesPostprocessCommand(parser), HermesProcessCommand(parser), + HermesReportCommand(parser), HermesVersionCommand(parser), ): if command.settings_class is not None: diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index 8bea1b6b..5d3280aa 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -192,11 +192,12 @@ def __call__(self, args: argparse.Namespace) -> None: # One pass must have been completed already. new_action = prov_doc.add_activity(data={ "schema:description": "merges the old data object with the new data", - "prov:used": [last_data.ref, new_data.ref], - "prov:wasInformedBy": [last_action.ref, new_action.ref], + "prov:used": [last_data.ref, new_data.ref, merged_strategies.ref], + "prov:wasInformedBy": [last_action.ref, new_action.ref, strategy_action.ref], "prov:wasAssociatedWith": process_command.ref }) # initial merge action of the merge - merged_doc.prov_objects = [new_action, new_data, last_data] # set the starting objects of the merge + # set the starting objects of the merge + merged_doc.prov_objects = [new_action, new_data, last_data, merged_strategies, strategy_action] self.log.info(f"### Merge data from {harvester} plugin") # merge data into the merge dict diff --git a/src/hermes/commands/process/invenio_merge.py b/src/hermes/commands/process/invenio_merge.py index 39e5c6ec..14feb484 100644 --- a/src/hermes/commands/process/invenio_merge.py +++ b/src/hermes/commands/process/invenio_merge.py @@ -26,9 +26,7 @@ def merge( value: Union[ld_merge_list, str], update: Union[BASIC_TYPE, TIME_TYPE, ld_dict, ld_list] ) -> ld_merge_list: - print(key, value, update) types = target.get("@type", []) - print(types) if key[-1] == iri["schema:license"] and (iri["schema:SoftwareSourceCode"] in types or iri["schema:SoftwareApplication"] in types): if len(value) == 1: if isinstance(value[0], str) or ( @@ -74,7 +72,6 @@ def merge( return [] target.reject(key, update) return value - print("fail") class InvenioProcessPlugin(HermesProcessPlugin): diff --git a/src/hermes/commands/report/__init__.py b/src/hermes/commands/report/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/hermes/commands/report/base.py b/src/hermes/commands/report/base.py new file mode 100644 index 00000000..0240a03e --- /dev/null +++ b/src/hermes/commands/report/base.py @@ -0,0 +1,313 @@ +# SPDX-FileCopyrightText: 2026 German Aerospace Center (DLR) +# +# SPDX-License-Identifier: Apache-2.0 + +# SPDX-FileContributor: Michael Fritzsche + +import argparse + +from pydantic import BaseModel + +from hermes.commands.base import HermesCommand +from hermes.model.context_manager import HermesContext +from hermes.model.provenance.ld_prov import ld_prov_list + + +class HermesReportSettings(BaseModel): + """Configuration of the ``report`` command.""" + pass + + +class HermesReportCommand(HermesCommand): + """ Gernate a summarized provenance report for the steps chosen by the user. """ + + command_name = "report" + settings_class = HermesReportSettings + + def init_command_parser(self, command_parser: argparse.ArgumentParser) -> None: + command_parser.add_argument( + "--steps", + nargs="*", + default=["harvest", "process", "curate", "deposit"], + choices=["harvest", "process", "curate", "deposit"], + help="Steps for which the report should be generated. Default is every step." + ) + + def __call__(self, args: argparse.Namespace) -> None: + print("\nProvenance report for HERMES:") + for step in args.steps: + ld_prov_list.INDICES = {} + match step: + case "harvest": + self.report_harvest() + case "process": + self.report_process() + case "curate": + self.report_curate() + case "deposit": + self.report_deposit() + print("") + + def report_harvest(self) -> None: + print("- Harvest:") + ctx = HermesContext() + ctx.prepare_step("harvest") + with ctx["provenance"] as cache: + try: + prov_doc = ld_prov_list.load_ld_prov_list(cache["result"]) + except KeyError: + print("No provenance data has been recorded so far.") + return + finally: + ctx.finalize_step("harvest") + harvest_base_plugin = prov_doc.get_hermes_base_plugin("harvest") + harvest_command = prov_doc.get_hermes_command("harvest") + hermes_cache = prov_doc.get_hermes_cache() + plugins = prov_doc.shallow_search(lambda node: ( + "prov:actedOnBehalfOf" in node and node["prov:actedOnBehalfOf"] == [harvest_base_plugin.ref] + )) + for plugin in plugins: + print( + f" - Plugin {plugin['@id'][24:]} ({plugin['schema:name'][0]}, version " + f"{vers if (vers := plugin.get('schema:softwareVersion', False)) else 'N/A'})" + ) + print(" - Loaded data from:") + for load_action in prov_doc.shallow_search(lambda node: ( + "prov:wasAssociatedWith" in node and + node["prov:wasAssociatedWith"] == [harvest_base_plugin.ref, plugin.ref] + )): + id_of_source = load_action["prov:used"][0]["@id"] + source = prov_doc.shallow_search(lambda node: ("@id" in node and node["@id"] == id_of_source))[0] + print( + f" - {source['schema:url'][0]} (at {load_action['prov:startedAtTime'][0]}, took " + + f"{load_action['prov:endedAtTime'][0]-load_action['prov:startedAtTime'][0]})" + ) + store_action = prov_doc.shallow_search(lambda node: ( + "prov:wasAssociatedWith" in node and + node["prov:wasAssociatedWith"] == [plugin.ref, hermes_cache.ref, harvest_command.ref] + ))[0] + print( + f" - Results stored (at {store_action['prov:startedAtTime'][0]}, took " + f"{store_action['prov:endedAtTime'][0]-store_action['prov:startedAtTime'][0]}) in:" + ) + for result in prov_doc.shallow_search(lambda node: ( + "prov:wasGeneratedBy" in node and node["prov:wasGeneratedBy"] == [store_action.ref] + )): + print(f" - {result['schema:url'][0]} ({result['schema:description'][0].split(' ')[1]})") + + def report_process(self) -> None: + print("- Process:") + ctx = HermesContext() + ctx.prepare_step("process") + with ctx["provenance"] as cache: + try: + prov_doc = ld_prov_list.load_ld_prov_list(cache["result"]) + except KeyError: + print("No provenance data has been recorded so far.") + return + finally: + ctx.finalize_step("process") + process_base_plugin = prov_doc.get_hermes_base_plugin("process") + plugins = prov_doc.shallow_search(lambda node: ( + "prov:actedOnBehalfOf" in node and node["prov:actedOnBehalfOf"] == [process_base_plugin.ref] + )) + for plugin in plugins: + print( + f" - Plugin {plugin['@id'][24:]} ({plugin['schema:name'][0]}, version " + f"{vers if (vers := plugin.get('schema:softwareVersion', False)) else 'N/A'}):" + ) + strategy_generation = prov_doc.shallow_search(lambda node: ( + "prov:wasAssociatedWith" in node and node["prov:wasAssociatedWith"] == [plugin.ref] + ))[0] + print( + f" - Generated strategies at {strategy_generation['prov:startedAtTime'][0]} took " + f"{strategy_generation['prov:endedAtTime'][0]-strategy_generation['prov:startedAtTime'][0]}" + ) + process_command = prov_doc.get_hermes_command("process") + hermes_cache = prov_doc.get_hermes_cache() + load_actions = prov_doc.shallow_search(lambda node: ( + "prov:wasAssociatedWith" in node and + node["prov:wasAssociatedWith"] == [hermes_cache.ref, process_command.ref] and + "prov:used" in node and + len(node["prov:used"]) == 3 + )) + for index, load_action in enumerate(sorted(load_actions, key=lambda it: it["prov:startedAtTime"][0]), start=1): + print( + f" - In load {index} loaded (at {load_action['prov:startedAtTime'][0]}, took" + f" {load_action['prov:endedAtTime'][0]-load_action['prov:startedAtTime'][0]}" + ", may have been overwritten) from:" + ) + loaded = [item["@id"] for item in load_action["prov:used"]] + sources = prov_doc.shallow_search(lambda node: ("@id" in node and node["@id"] in loaded)) + for source in sources: + print(f" - {source['schema:url'][0]}") + bigest_mergers = prov_doc.shallow_search(lambda node: ( + "prov:wasAssociatedWith" in node and + node["prov:wasAssociatedWith"] == [process_command.ref] and + "prov:wasInformedBy" in node and + len(node["prov:wasInformedBy"]) == 3 + )) + for index, merger in enumerate(sorted(bigest_mergers, key=lambda it: it["prov:startedAtTime"][0]), start=1): + if index == 1: + merged = "merged data from load 1 with data of load 2" + else: + merged = f"merged data from load {index + 1} with old results" + print( + f" - Merge {index} {merged} at {merger['prov:startedAtTime'][0]} took " + f"{merger['prov:endedAtTime'][0]-merger['prov:startedAtTime'][0]}" + ) + write_action = prov_doc.shallow_search(lambda node: ( + "prov:wasAssociatedWith" in node and + node["prov:wasAssociatedWith"] == [hermes_cache.ref, process_command.ref] and + "prov:used" in node and + len(node["prov:used"]) == 1 + ))[0] + stored_objects = prov_doc.shallow_search(lambda node: ( + "prov:wasGeneratedBy" in node and node["prov:wasGeneratedBy"] == [write_action.ref] + )) + print( + f" - Results stored (at {write_action['prov:startedAtTime'][0]} took " + f"{write_action['prov:endedAtTime'][0]-write_action['prov:startedAtTime'][0]}) in:" + ) + for res in stored_objects: + print(f" - {res['schema:url'][0]} ({res['schema:description'][0].split(' ')[1]})") + + def report_curate(self) -> None: + print("- Curate:") + ctx = HermesContext() + ctx.prepare_step("curate") + with ctx["provenance"] as cache: + try: + prov_doc = ld_prov_list.load_ld_prov_list(cache["result"]) + except KeyError: + print("No provenance data has been recorded so far.") + return + finally: + ctx.finalize_step("curate") + curate_base_plugin = prov_doc.get_hermes_base_plugin("curate") + curate_plugin = prov_doc.shallow_search(lambda node: ( + "prov:actedOnBehalfOf" in node and node["prov:actedOnBehalfOf"] == [curate_base_plugin.ref] + ))[0] + print( + f" - Plugin used:\n - {curate_plugin['@id'][23:]} ({curate_plugin['schema:name'][0]}, version " + f"{vers if (vers := curate_plugin.get('schema:softwareVersion', False)) else 'N/A'})" + ) + process_command = prov_doc.get_hermes_command("process") + hermes_cache = prov_doc.get_hermes_cache() + store_action_of_process = prov_doc.shallow_search(lambda node: ( + "prov:wasAssociatedWith" in node and + node["prov:wasAssociatedWith"] == [process_command.ref, hermes_cache.ref] and + "prov:wasInformedBy" in node + ))[0] + stored_results_of_process = prov_doc.shallow_search(lambda node: ( + "prov:wasGeneratedBy" in node and node["prov:wasGeneratedBy"] == [store_action_of_process.ref] + )) + load_action = prov_doc.shallow_search(lambda node: ( + "prov:used" in node and node["prov:used"] == [res.ref for res in stored_results_of_process] + ))[0] + curate_activity = prov_doc.shallow_search(lambda node: ( + "prov:wasInfluencedBy" in node and node["prov:wasInfluencedBy"] == [curate_plugin.ref] + ))[0] + results = prov_doc.shallow_search(lambda node: ( + "prov:wasDerivedFrom" in node and node["prov:wasDerivedFrom"] == [curate_activity.ref] + )) + write = prov_doc.shallow_search(lambda node: ( + "prov:used" in node and node["prov:used"] == [curate_activity.ref] + ))[0] + print( + f" - Time consumed:\n - Curation at ~{load_action['prov:endedAtTime'][0]} took" + f" ~{curate_activity['prov:generatedAtTime'][0]-load_action['prov:endedAtTime'][0]}" + ) + print( + f" - Uncurated metadata loaded (at {load_action['prov:startedAtTime'][0]}" + f", took {load_action['prov:endedAtTime'][0]-load_action['prov:startedAtTime'][0]}" + ", may have been overwritten) from:" + ) + for source in stored_results_of_process: + print(4*" " + f"- {source['schema:url'][0]} ({source['schema:description'][0].split(' ')[1]})") + print( + f" - Curated metadata stored (at {write['prov:startedAtTime'][0]}, took " + f"{write['prov:endedAtTime'][0]-write['prov:startedAtTime'][0]}) in:" + ) + for result in results: + print(f" - {result['schema:url'][0]} ({result['schema:description'][0].split(' ')[1]})") + + def report_deposit(self) -> None: + print("- Deposit:") + ctx = HermesContext() + ctx.prepare_step("deposit") + with ctx["provenance"] as cache: + try: + prov_doc = ld_prov_list.load_ld_prov_list(cache["result"]) + except KeyError: + print("No provenance data has been recorded so far.") + return + finally: + ctx.finalize_step("deposit") + deposit_base_plugin = prov_doc.get_hermes_base_plugin("deposit") + deposit_plugin = prov_doc.shallow_search(lambda node: ( + "prov:actedOnBehalfOf" in node and node["prov:actedOnBehalfOf"] == [deposit_base_plugin.ref] + ))[0] + print( + f" - Plugin used:\n - {deposit_plugin['@id'][24:]} ({deposit_plugin['schema:name'][0]}, version " + f"{vers if (vers := deposit_plugin.get('schema:softwareVersion', False)) else 'N/A'})" + ) + curate_command = prov_doc.get_hermes_command("curate") + hermes_cache = prov_doc.get_hermes_cache() + store_action_of_curate = prov_doc.shallow_search(lambda node: ( + "prov:wasAssociatedWith" in node and + node["prov:wasAssociatedWith"] == [curate_command.ref, hermes_cache.ref] and + "prov:used" in node and + len(node["prov:used"]) == 1 + ))[0] + stored_results_of_curate = prov_doc.shallow_search(lambda node: ( + "prov:wasGeneratedBy" in node and node["prov:wasGeneratedBy"] == [store_action_of_curate.ref] + )) + load_action = prov_doc.shallow_search(lambda node: ( + "prov:used" in node and node["prov:used"] == [res.ref for res in stored_results_of_curate] + ))[0] + mapped_metadata = prov_doc.shallow_search(lambda node: ( + "prov:wasAttributedTo" in node and node["prov:wasAttributedTo"] == [deposit_plugin.ref] + ))[0] + store_mapped = prov_doc.shallow_search(lambda node: ( + "prov:used" in node and node["prov:used"] == [mapped_metadata.ref] + ))[0] + result_mapped = prov_doc.shallow_search(lambda node: ( + "prov:wasGeneratedBy" in node and node["prov:wasGeneratedBy"] == [store_mapped.ref] + ))[0] + updated_metadata = prov_doc.shallow_search(lambda node: ( + "prov:wasInfluencedBy" in node and node["prov:wasInfluencedBy"] == [deposit_plugin.ref] + ))[0] + result_updated = prov_doc.shallow_search(lambda node: ( + "prov:wasDerivedFrom" in node and node["prov:wasDerivedFrom"] == [updated_metadata.ref] + ))[0] + store_updated = prov_doc.shallow_search(lambda node: ( + "prov:used" in node and node["prov:used"] == [updated_metadata.ref] + ))[0] + map_action = prov_doc.shallow_search(lambda node: ( + "@id" in node and node["@id"] == mapped_metadata["prov:wasGeneratedBy"][0]["@id"] + ))[0] + print( + " - Time consumed:\n" + f" - Preparation at ~{load_action['prov:endedAtTime'][0]} took ~" + f"{map_action['prov:startedAtTime'][0]-load_action['prov:endedAtTime'][0]}\n" + f" - Mapping at ~{map_action['prov:startedAtTime'][0]} took ~" + f"{map_action['prov:endedAtTime'][0]-map_action['prov:startedAtTime'][0]}\n" + f" - Creating new or initial version and updating metadata at ~{store_mapped['prov:endedAtTime'][0]}" + f" took ~{updated_metadata['prov:generatedAtTime'][0]-store_mapped['prov:endedAtTime'][0]}\n" + f" - Deletion of artifacts, upload of artifacts and publication at" + f" ~{store_updated['prov:endedAtTime'][0]} took N/A\n" + f" - Curated metadata loaded (at {load_action['prov:startedAtTime'][0]}" + f", took {load_action['prov:endedAtTime'][0]-load_action['prov:startedAtTime'][0]}" + ", may have been overwritten) from:" + ) + for source in stored_results_of_curate: + print(4*" " + f"- {source['schema:url'][0]} ({source['schema:description'][0].split(' ')[1]})") + print( + f" - Metadata mapped for deposit stored (at {store_mapped['prov:startedAtTime'][0]}, took " + f"{store_mapped['prov:endedAtTime'][0]-store_mapped['prov:startedAtTime'][0]}) in:\n" + f" - {result_mapped['schema:url'][0]}\n" + f" - Metadata updated after deposit stored (at {store_updated['prov:startedAtTime'][0]}, took " + f"{store_updated['prov:endedAtTime'][0]-store_updated['prov:startedAtTime'][0]}) in:\n" + f" - {result_updated['schema:url'][0]}" + ) diff --git a/src/hermes/model/merge/container.py b/src/hermes/model/merge/container.py index fc77fdcc..3a536a72 100644 --- a/src/hermes/model/merge/container.py +++ b/src/hermes/model/merge/container.py @@ -250,8 +250,12 @@ def __setitem__(self: Self, key: str, value: Union[JSON_LD_VALUE, BASIC_TYPE, TI "schema:name": f"merge values at {str(self.path+[key])}", "schema:description": f"Inserting value in the second 'used' value at {str(self.path+[key])} into the " "first 'used' value at the same point, no merger needed.", - "prov:used": {"@list": [self.prov_objects[2].ref, self.prov_objects[1].ref]}, - "prov:wasInformedBy": self.prov_objects[0].ref + "prov:used": {"@list": [ + self.prov_objects[2].ref, + self.prov_objects[1].ref, + self.prov_objects[3].ref + ]}, + "prov:wasInformedBy": {"@list": [self.prov_objects[0].ref, self.prov_objects[4].ref]} }) create_new_merged_data = True # update the entry of self[key] @@ -346,9 +350,10 @@ def _merge_item( "prov:wasAssociatedWith": self.prov_doc.get_hermes_command("process").ref, "prov:used": {"@list": [ self.prov_objects[2].ref, - self.prov_objects[1].ref + self.prov_objects[1].ref, + self.prov_objects[3].ref ]}, - "prov:wasInformedBy": self.prov_objects[0].ref + "prov:wasInformedBy": {"@list": [self.prov_objects[0].ref, self.prov_objects[4].ref]} }) self.prov_objects[0] = merge_activity else: From b3fc205abc94f59444c337bb80c32bc117f76caa Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Wed, 19 Aug 2026 17:20:57 +0200 Subject: [PATCH 22/41] flake8 --- src/hermes/commands/process/invenio_merge.py | 14 ++++++++++---- src/hermes/model/merge/container.py | 5 ++--- src/hermes/model/provenance/ld_prov.py | 2 +- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/hermes/commands/process/invenio_merge.py b/src/hermes/commands/process/invenio_merge.py index 14feb484..f5e27034 100644 --- a/src/hermes/commands/process/invenio_merge.py +++ b/src/hermes/commands/process/invenio_merge.py @@ -27,7 +27,10 @@ def merge( update: Union[BASIC_TYPE, TIME_TYPE, ld_dict, ld_list] ) -> ld_merge_list: types = target.get("@type", []) - if key[-1] == iri["schema:license"] and (iri["schema:SoftwareSourceCode"] in types or iri["schema:SoftwareApplication"] in types): + if ( + key[-1] == iri["schema:license"] and + (iri["schema:SoftwareSourceCode"] in types or iri["schema:SoftwareApplication"] in types) + ): if len(value) == 1: if isinstance(value[0], str) or ( isinstance(value[0], (dict, ld_merge_dict)) and [*value[0].keys()] == ["@id"] @@ -43,9 +46,11 @@ def merge( return update target.reject(key, update) return value - if ((key[-1] == iri["schema:familyName"] and iri["schema:Person"] in types) or + if ( + (key[-1] == iri["schema:familyName"] and iri["schema:Person"] in types) or (key[-1] == iri["schema:name"] and iri["schema:Person"] in types) or - (key[-1] == iri["schema:name"] and (iri["schema:SoftwareSourceCode"] in types or iri["schema:SoftwareApplication"] in types)) + (key[-1] == iri["schema:name"] and (iri["schema:SoftwareSourceCode"] in types or + iri["schema:SoftwareApplication"] in types)) ): if len(value) == 1: if value != update: @@ -58,7 +63,8 @@ def merge( return value target.reject(key, update) return value - if ((key[-1] == iri["schema:version"] or key[-1] == iri["schema:description"]) and + if ( + (key[-1] == iri["schema:version"] or key[-1] == iri["schema:description"]) and (iri["schema:SoftwareSourceCode"] in types or iri["schema:SoftwareApplication"] in types) ): if len(value) == 1: diff --git a/src/hermes/model/merge/container.py b/src/hermes/model/merge/container.py index 3a536a72..64cb7ac0 100644 --- a/src/hermes/model/merge/container.py +++ b/src/hermes/model/merge/container.py @@ -249,7 +249,7 @@ def __setitem__(self: Self, key: str, value: Union[JSON_LD_VALUE, BASIC_TYPE, TI merge_activity = self.prov_doc.add_activity(data={ "schema:name": f"merge values at {str(self.path+[key])}", "schema:description": f"Inserting value in the second 'used' value at {str(self.path+[key])} into the " - "first 'used' value at the same point, no merger needed.", + "first 'used' value at the same point, no merger needed.", "prov:used": {"@list": [ self.prov_objects[2].ref, self.prov_objects[1].ref, @@ -269,7 +269,7 @@ def __setitem__(self: Self, key: str, value: Union[JSON_LD_VALUE, BASIC_TYPE, TI self.prov_objects[0] = merge_activity if create_new_merged_data: outer_most_parent = self - while outer_most_parent.parent != None: + while outer_most_parent.parent is not None: outer_most_parent = outer_most_parent.parent self.prov_objects[2] = self.prov_doc.add_entity(data={ "@type": "schema:CreativeWork", @@ -336,7 +336,6 @@ def _merge_item( type_of_used_strategy = ld_type key_of_used_strategy = key - # choose one merge strategy and return the item returned by following the merge startegy merger = strategy.get(key, strategy.get(None, None)) if merger is None: diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index ba86c274..c8855eac 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -11,7 +11,7 @@ from hermes import utils from hermes.commands.base import HermesCommand, HermesPlugin from hermes.model.types import ld_dict, ld_list -from hermes.model.types.ld_container import BASIC_TYPE, EXPANDED_JSON_LD_VALUE, JSON_LD_CONTEXT_DICT +from hermes.model.types.ld_container import EXPANDED_JSON_LD_VALUE, JSON_LD_CONTEXT_DICT from hermes.model.types.ld_context import ALL_CONTEXTS, iri_map From 537bb2166e8f58185cab363e09084c0c8f28028c Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Thu, 20 Aug 2026 16:45:57 +0200 Subject: [PATCH 23/41] add provenance recording for postprocess --- src/hermes/commands/deposit/base.py | 11 +- src/hermes/commands/harvest/base.py | 5 +- src/hermes/commands/postprocess/base.py | 162 ++++++++++++++++++ src/hermes/commands/postprocess/invenio.py | 31 +--- .../commands/postprocess/invenio_rdm.py | 11 +- src/hermes/model/provenance/ld_prov.py | 2 +- 6 files changed, 181 insertions(+), 41 deletions(-) diff --git a/src/hermes/commands/deposit/base.py b/src/hermes/commands/deposit/base.py index b8beb1c5..0d2c6cdc 100644 --- a/src/hermes/commands/deposit/base.py +++ b/src/hermes/commands/deposit/base.py @@ -4,6 +4,7 @@ # SPDX-FileContributor: David Pape # SPDX-FileContributor: Michael Meinel +# SPDX-FileContributor: Michael Fritzsche import abc import argparse @@ -82,9 +83,8 @@ def __call__(self, command: HermesCommand, prov_doc: ld_prov_list) -> None: deposit = self.map_metadata() end_of_map = datetime.datetime.now() with self.ctx[target] as cache: - start_of_store = datetime.datetime.now() cache["deposit"] = deposit - end_of_store = datetime.datetime.now() + end_of_store = datetime.datetime.now() if prov_doc is not None: map_action = prov_doc.add_activity(data={ @@ -107,7 +107,7 @@ def __call__(self, command: HermesCommand, prov_doc: ld_prov_list) -> None: "schema:description": "Stores the mapped metadata.", "prov:used": mapped_data.ref, "prov:wasAssociatedWith": [hermes_cache.ref, deposit_command.ref, deposit_base_plugin.ref], - "prov:startedAtTime": start_of_store, + "prov:startedAtTime": end_of_map, "prov:endedAtTime": end_of_store }) prov_doc.add_entity(data={ @@ -130,9 +130,8 @@ def __call__(self, command: HermesCommand, prov_doc: ld_prov_list) -> None: updated_deposit = self.update_metadata() end_of_update_map = datetime.datetime.now() with self.ctx[target] as cache: - start_of_second_store = datetime.datetime.now() cache["result"] = updated_deposit - end_of_second_store = datetime.datetime.now() + end_of_second_store = datetime.datetime.now() self.ctx.finalize_step("deposit") if prov_doc is not None: @@ -148,7 +147,7 @@ def __call__(self, command: HermesCommand, prov_doc: ld_prov_list) -> None: "schema:description": "Stores the mapped metadata.", "prov:used": updated_mapped_data.ref, "prov:wasAssociatedWith": [hermes_cache.ref, deposit_command.ref, deposit_base_plugin.ref], - "prov:startedAtTime": start_of_second_store, + "prov:startedAtTime": end_of_update_map, "prov:endedAtTime": end_of_second_store }) prov_doc.add_entity(data={ diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index ec02f948..7395e192 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileContributor: Michael Meinel +# SPDX-FileContributor: Michael Fritzsche import argparse import datetime @@ -54,10 +55,6 @@ def load(self, func, source, *args, **kwargs): self.io_operations.append((source_metadata, io_operation, loaded_metadata)) return result - def write(): - # TODO: Is this needed? If yes, it needs to be implemented - pass - class HarvestSettings(BaseModel): """Generic harvesting settings.""" diff --git a/src/hermes/commands/postprocess/base.py b/src/hermes/commands/postprocess/base.py index 99a26d73..2a105d20 100644 --- a/src/hermes/commands/postprocess/base.py +++ b/src/hermes/commands/postprocess/base.py @@ -6,19 +6,93 @@ # SPDX-FileContributor: Michael Fritzsche import argparse +import datetime +from io import IOBase +from pathlib import Path +from typing import Any, Callable, Optional from pydantic import BaseModel from hermes.commands.base import HermesCommand, HermesPlugin from hermes.error import HermesPluginRunError +from hermes.model.context_manager import HermesContext +from hermes.model.provenance.ld_prov import ld_prov_list class HermesPostprocessPlugin(HermesPlugin): """ Base plugin for postprocess plugins. """ + def __init__(self): + self.cache_operations: list[tuple[str, dict, dict]] = [] + self.load_operations: list[tuple[dict, dict, dict]] = [] + self.write_operations: list[tuple[dict, dict, dict]] = [] + super().__init__() + def __call__(self, command: HermesCommand) -> None: pass + def get_deposit_result(self, target: str) -> dict: + source_metadata = target[:] + load_operation = {"schema:description": f"loads the result of deposit plugin {target}"} + ctx = HermesContext() + ctx.prepare_step("deposit") + load_operation["prov:startedAtTime"] = datetime.datetime.now() + with ctx[target] as cache: + res = cache["result"] + load_operation["prov:endedAtTime"] = datetime.datetime.now() + ctx.finalize_step("deposit") + loaded_data = {"schema:description": "the loaded data", "schema:text": str(res)} + self.cache_operations.append((source_metadata, load_operation, loaded_data)) + return res + + def load(self, func: Callable, source: Any, *args, **kwargs) -> Any: + source_metadata = {"schema:description": "metadata source"} + if isinstance(source, IOBase): + source_metadata["schema:url"] = Path(source.name).absolute().as_uri() + elif isinstance(source, Path): + source_metadata["schema:url"] = source.absolute().as_uri() + elif isinstance(source, str): + try: + source_metadata["schema:url"] = Path(source).absolute().as_uri() + except Exception: + source_metadata["schema:url"] = source + load_operation = { + "schema:description": "Load operation called with (" + f"{source_metadata['schema:url'] if 'schema:url' in source_metadata else str(source)}" + f"{', ' + str(args) if args else ''}{', ' + str(kwargs) if kwargs else ''}).", + "schema:name": f"{func.__module__}.{func.__qualname__}" + } + load_operation["prov:startedAtTime"] = datetime.datetime.now() + result = func(source, *args, **kwargs) + load_operation["prov:endedAtTime"] = datetime.datetime.now() + loaded_metadata = {"schema:description": "the loaded data", "schema:text": str(result)} + self.load_operations.append((source_metadata, load_operation, loaded_metadata)) + return result + + def write(self, func: Callable, data: Any, destination: Any, *args, **kwargs) -> Any: + destination_metadata = {"schema:description": "metadata destination"} + if isinstance(destination, IOBase): + destination_metadata["schema:url"] = Path(destination.name).absolute().as_uri() + elif isinstance(destination, Path): + destination_metadata["schema:url"] = destination.absolute().as_uri() + elif isinstance(destination, str): + try: + destination_metadata["schema:url"] = Path(destination).absolute().as_uri() + except Exception: + destination_metadata["schema:url"] = destination + write_operation = { + "schema:description": f"Write operation called with ({str(data)}," + f"{destination_metadata['schema:url'] if 'schema:url' in destination_metadata else str(destination)}" + f"{', ' + str(args) if args else ''}{', ' + str(kwargs) if kwargs else ''}).", + "schema:name": f"{func.__module__}.{func.__qualname__}" + } + write_operation["prov:startedAtTime"] = datetime.datetime.now() + result = func(data, destination, *args, **kwargs) + write_operation["prov:endedAtTime"] = datetime.datetime.now() + written_metadata = {"schema:description": "the written data", "schema:text": str(data)} + self.write_operations.append((written_metadata, write_operation, destination_metadata)) + return result + class PostprocessSettings(BaseModel): """Generic post-processing settings.""" @@ -36,6 +110,13 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info("# Postprocessing") self.args = args plugin_names = self.settings.run + prov_doc = self.load_prov_doc() + if prov_doc is not None: + prov_doc.add_hermes_settings(self) + prov_doc.add_settings_to_command("postprocess", self) + hermes_cache = prov_doc.get_hermes_cache() + postprocess_command = prov_doc.get_hermes_command("postprocess") + postprocess_base_plugin = prov_doc.get_hermes_base_plugin("postprocess") if not plugin_names: self.log.warning("# No plugin was configured to be run yet the postprocess command was executed.") @@ -62,6 +143,87 @@ def __call__(self, args: argparse.Namespace) -> None: ran_any = True + if prov_doc is None: + continue + + plugin = prov_doc.add_hermes_plugin("postprocess", plugin_name, plugin_func, self) + cache_loads = plugin_func.cache_operations + loads = plugin_func.load_operations + writes = plugin_func.write_operations + load_actions, loaded_datas = [], [] + for cache_load in cache_loads: + deposit_plugin = prov_doc.get_hermes_plugin("postprocess", cache_load[0]) + updated_metadata = prov_doc.shallow_search(lambda node: ( + "prov:wasInfluencedBy" in node and node["prov:wasInfluencedBy"] == [deposit_plugin.ref] + ))[0] + updated_metadata = prov_doc.shallow_search(lambda node: ( + "prov:wasDerivedFrom" in node and node["prov:wasDerivedFrom"] == [updated_metadata.ref] + ))[0] + load_actions.append(prov_doc.add_activity(data=cache_load[1])) + load_actions[-1].update({ + "prov:used": updated_metadata.ref, + "prov:wasAssociatedWith": [ + plugin.ref, postprocess_base_plugin.ref, postprocess_command.ref, hermes_cache.ref + ] + }) + loaded_datas.append(prov_doc.add_entity(data=cache_load[2])) + loaded_datas[-1].update({ + "prov:wasGeneratedBy": load_actions[-1].ref, + "prov:wasDerivedFrom": updated_metadata.ref, + "prov:wasAttributedTo": hermes_cache.ref + }) + for load in loads: + source = prov_doc.add_entity(data=load[0]) + load_actions.append(prov_doc.add_activity(data=load[1])) + load_actions[-1].update({ + "prov:used": source.ref, + "prov:wasAssociatedWith": [plugin.ref, postprocess_base_plugin.ref, postprocess_command.ref] + }) + loaded_datas.append(prov_doc.add_entity(data=load[2])) + loaded_datas[-1].update({ + "prov:wasGeneratedBy": load_actions[-1].ref, + "prov:wasDerivedFrom": source.ref, + "prov:wasAttributedTo": [plugin.ref, postprocess_base_plugin.ref, postprocess_command.ref] + }) + load_actions = [load_action.ref for load_action in load_actions] + loaded_datas = [loaded_data.ref for loaded_data in loaded_datas] + for write in writes: + data = prov_doc.add_entity(data=write[0]) + data.update({"prov:wasDerivedFrom": loaded_datas, "prov:wasInfluencedBy": plugin.ref}) + write_action = prov_doc.add_activity(data=write[1]) + write_action.update({ + "prov:used": data.ref, + "prov:wasAssociatedWith": [plugin.ref, postprocess_base_plugin.ref, postprocess_command.ref] + }) + written_data = prov_doc.add_entity(data=write[2]) + written_data.update({ + "prov:wasGeneratedBy": write_action.ref, + "prov:wasDerivedFrom": data.ref, + "prov:wasAttributedTo": [plugin.ref, postprocess_base_plugin.ref, postprocess_command.ref] + }) + + if prov_doc is not None: + ctx = HermesContext() + ctx.prepare_step("postprocess") + with ctx["provenance"] as cache: + cache["result"] = prov_doc.ld_value + ctx.finalize_step("postprocess") + if not ran_any: self.log.critical("## No postprocess plugin ran successfully.") raise HermesPluginRunError("No postprocess plugin ran successfully.") + + def load_prov_doc(self) -> Optional[ld_prov_list]: + ctx = HermesContext() + ctx.prepare_step("deposit") + with ctx["provenance"] as cache: + try: + return ld_prov_list.load_ld_prov_list(cache["result"]) + except Exception: + self.log.warning( + "The provenance data from the deposit step could not be loaded. " + "Postprocessing will proceed without collecting provenance data.", + exc_info=1 + ) + finally: + ctx.finalize_step("deposit") diff --git a/src/hermes/commands/postprocess/invenio.py b/src/hermes/commands/postprocess/invenio.py index 5c0de3e6..f260b306 100644 --- a/src/hermes/commands/postprocess/invenio.py +++ b/src/hermes/commands/postprocess/invenio.py @@ -13,7 +13,6 @@ import tomlkit from hermes.error import MisconfigurationError -from hermes.model.context_manager import HermesContext from ..base import HermesCommand from .base import HermesPostprocessPlugin @@ -23,13 +22,9 @@ class config_record_id(HermesPostprocessPlugin): def __call__(self, command: HermesCommand): - ctx = HermesContext() - ctx.prepare_step("deposit") - with ctx["invenio"] as manager: - deposition = manager["result"] - ctx.finalize_step("deposit") + deposition = self.get_deposit_result("invenio") - conf = tomlkit.load(open('hermes.toml', 'r')) + conf = self.load(tomlkit.load, open('hermes.toml', 'r')) try: old_record_id = conf["deposit"]["invenio"]["record_id"] if old_record_id == deposition["record_id"]: @@ -42,16 +37,12 @@ def __call__(self, command: HermesCommand): except KeyError: pass conf.setdefault("deposit", {}).setdefault("invenio", {})["record_id"] = deposition['record_id'] - tomlkit.dump(conf, open('hermes.toml', 'w')) + self.write(tomlkit.dump, conf, open('hermes.toml', 'w')) class cff_doi(HermesPostprocessPlugin): def __call__(self, command: HermesCommand): - ctx = HermesContext() - ctx.prepare_step("deposit") - with ctx["invenio"] as manager: - deposition = manager["result"] - ctx.finalize_step("deposit") + deposition = self.get_deposit_result("invenio") yaml = YAML() yaml.default_flow_style = False @@ -60,7 +51,7 @@ def __call__(self, command: HermesCommand): yaml.allow_unicode = True try: - cff = yaml.load(open('CITATION.cff', 'r')) + cff = self.load(yaml.load, open('CITATION.cff', 'r')) new_identifier = { 'description': f"DOI for the published version {deposition['metadata']['version']} " "[generated by hermes]", @@ -71,22 +62,18 @@ def __call__(self, command: HermesCommand): cff['identifiers'].append(new_identifier) else: cff['identifiers'] = [new_identifier] - yaml.dump(cff, open('CITATION.cff', 'w')) + self.write(yaml.dump, cff, open('CITATION.cff', 'w')) except Exception as e: raise RuntimeError("Update of CITATION.cff failed.") from e class codemeta_doi(HermesPostprocessPlugin): def __call__(self, command: HermesCommand): - ctx = HermesContext() - ctx.prepare_step("deposit") - with ctx["invenio"] as manager: - deposition = manager["result"] - ctx.finalize_step("deposit") + deposition = self.get_deposit_result("invenio") try: with open("codemeta.json", "r") as file: - codemeta = json.load(file) + codemeta = self.load(json.load, file) if "@id" not in codemeta: codemeta["@id"] = deposition['doi'] if "referencePublication" not in codemeta: @@ -96,6 +83,6 @@ def __call__(self, command: HermesCommand): else: codemeta["referencePublication"] = [codemeta["referencePublication"], deposition['doi']] with open("codemeta.json", "w") as file: - json.dump(codemeta, file) + self.write(json.dump, codemeta, file) except Exception as e: raise RuntimeError("Update of CITATION.cff failed.") from e diff --git a/src/hermes/commands/postprocess/invenio_rdm.py b/src/hermes/commands/postprocess/invenio_rdm.py index afee8dd2..ff9fc549 100644 --- a/src/hermes/commands/postprocess/invenio_rdm.py +++ b/src/hermes/commands/postprocess/invenio_rdm.py @@ -11,7 +11,6 @@ import tomlkit from hermes.error import MisconfigurationError -from hermes.model.context_manager import HermesContext from ..base import HermesCommand from .base import HermesPostprocessPlugin @@ -21,13 +20,9 @@ class config_record_id(HermesPostprocessPlugin): def __call__(self, command: HermesCommand): - ctx = HermesContext() - ctx.prepare_step("deposit") - with ctx["invenio_rdm"] as manager: - deposition = manager["result"] - ctx.finalize_step("deposit") + deposition = self.get_deposit_result("invenio_rdm") - conf = tomlkit.load(open('hermes.toml', 'r')) + conf = self.load(tomlkit.load, open('hermes.toml', 'r')) try: old_record_id = conf["deposit"]["invenio_rdm"]["record_id"] if old_record_id == deposition["record_id"]: @@ -40,4 +35,4 @@ def __call__(self, command: HermesCommand): except KeyError: pass conf.setdefault("deposit", {}).setdefault("invenio_rdm", {})["record_id"] = deposition['record_id'] - tomlkit.dump(conf, open('hermes.toml', 'w')) + self.write(tomlkit.dump, conf, open('hermes.toml', 'w')) diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index c8855eac..3d5d89f8 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -218,7 +218,7 @@ def add_hermes_plugin(self, step: str, name: str, plugin: HermesPlugin, command: except Exception: del data["schema:supportingData"] try: - data["schema:softwareVersion"] = metadata(plugin.__module__)["version"] + data["schema:softwareVersion"] = metadata(plugin.__module__.split(".")[0])["version"] except Exception: pass node = self.add_agent(data=data) From 28f6edee94a5efec53139f5248484c174723a5ea Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Tue, 25 Aug 2026 12:05:40 +0200 Subject: [PATCH 24/41] implemented postprocess report --- src/hermes/commands/postprocess/base.py | 3 +- src/hermes/commands/report/base.py | 85 ++++++++++++++++++++++--- 2 files changed, 78 insertions(+), 10 deletions(-) diff --git a/src/hermes/commands/postprocess/base.py b/src/hermes/commands/postprocess/base.py index 2a105d20..d569a2b7 100644 --- a/src/hermes/commands/postprocess/base.py +++ b/src/hermes/commands/postprocess/base.py @@ -195,8 +195,7 @@ def __call__(self, args: argparse.Namespace) -> None: "prov:used": data.ref, "prov:wasAssociatedWith": [plugin.ref, postprocess_base_plugin.ref, postprocess_command.ref] }) - written_data = prov_doc.add_entity(data=write[2]) - written_data.update({ + prov_doc.add_entity(data=write[2]).update({ "prov:wasGeneratedBy": write_action.ref, "prov:wasDerivedFrom": data.ref, "prov:wasAttributedTo": [plugin.ref, postprocess_base_plugin.ref, postprocess_command.ref] diff --git a/src/hermes/commands/report/base.py b/src/hermes/commands/report/base.py index 0240a03e..abc0ad15 100644 --- a/src/hermes/commands/report/base.py +++ b/src/hermes/commands/report/base.py @@ -5,6 +5,7 @@ # SPDX-FileContributor: Michael Fritzsche import argparse +from typing_extensions import Self from pydantic import BaseModel @@ -24,16 +25,16 @@ class HermesReportCommand(HermesCommand): command_name = "report" settings_class = HermesReportSettings - def init_command_parser(self, command_parser: argparse.ArgumentParser) -> None: + def init_command_parser(self: Self, command_parser: argparse.ArgumentParser) -> None: command_parser.add_argument( "--steps", nargs="*", - default=["harvest", "process", "curate", "deposit"], - choices=["harvest", "process", "curate", "deposit"], + default=["harvest", "process", "curate", "deposit", "postprocess"], + choices=["harvest", "process", "curate", "deposit", "postprocess"], help="Steps for which the report should be generated. Default is every step." ) - def __call__(self, args: argparse.Namespace) -> None: + def __call__(self: Self, args: argparse.Namespace) -> None: print("\nProvenance report for HERMES:") for step in args.steps: ld_prov_list.INDICES = {} @@ -46,9 +47,11 @@ def __call__(self, args: argparse.Namespace) -> None: self.report_curate() case "deposit": self.report_deposit() + case "postprocess": + self.report_postprocess() print("") - def report_harvest(self) -> None: + def report_harvest(self: Self) -> None: print("- Harvest:") ctx = HermesContext() ctx.prepare_step("harvest") @@ -95,7 +98,7 @@ def report_harvest(self) -> None: )): print(f" - {result['schema:url'][0]} ({result['schema:description'][0].split(' ')[1]})") - def report_process(self) -> None: + def report_process(self: Self) -> None: print("- Process:") ctx = HermesContext() ctx.prepare_step("process") @@ -172,7 +175,7 @@ def report_process(self) -> None: for res in stored_objects: print(f" - {res['schema:url'][0]} ({res['schema:description'][0].split(' ')[1]})") - def report_curate(self) -> None: + def report_curate(self: Self) -> None: print("- Curate:") ctx = HermesContext() ctx.prepare_step("curate") @@ -232,7 +235,7 @@ def report_curate(self) -> None: for result in results: print(f" - {result['schema:url'][0]} ({result['schema:description'][0].split(' ')[1]})") - def report_deposit(self) -> None: + def report_deposit(self: Self) -> None: print("- Deposit:") ctx = HermesContext() ctx.prepare_step("deposit") @@ -311,3 +314,69 @@ def report_deposit(self) -> None: f"{store_updated['prov:endedAtTime'][0]-store_updated['prov:startedAtTime'][0]}) in:\n" f" - {result_updated['schema:url'][0]}" ) + + def report_postprocess(self: Self) -> None: + print("- Postprocess:") + ctx = HermesContext() + ctx.prepare_step("postprocess") + with ctx["provenance"] as cache: + try: + prov_doc = ld_prov_list.load_ld_prov_list(cache["result"]) + except KeyError: + print("No provenance data has been recorded so far.") + return + finally: + ctx.finalize_step("postprocess") + cache = prov_doc.get_hermes_cache() + command = prov_doc.get_hermes_command("postprocess") + base_plugin = prov_doc.get_hermes_base_plugin("postprocess") + plugin = prov_doc.shallow_search(lambda node: ( + "prov:actedOnBehalfOf" in node and node["prov:actedOnBehalfOf"] == [base_plugin.ref] + ))[0] + print( + f" - Plugin used:\n - {plugin['@id'][28]} ({plugin['schema:name'][0]}, version " + f"{vers if (vers := plugin.get('schema:softwareVersion', False)) else 'N/A'})" + ) + cache_loads = prov_doc.shallow_search(lambda node: ( + "prov:wasAssociatedWith" in node and + node["prov:wasAssociatedWith"] == [plugin.ref, base_plugin.ref, command.ref, cache.ref] + )) + print(" - Used deposit results:") + for index, cache_load in enumerate(cache_loads, start=1): + source_id = cache_load["prov:used"][0]["@id"] + source = prov_doc.shallow_search(lambda node: ("@id" in node and node["@id"] == [source_id]))[0] + print( + f" - Load {index} at {cache_load['prov:startedAtTime']} took " + f"{cache_load['prov:endedAtTime']-cache_load['prov:startedAtTime']} from:\n" + f" - {source['schema:url']}" + ) + io_ops = prov_doc.shallow_search(lambda node: ( + "prov:wasAssociatedWith" in node and + node["prov:wasAssociatedWith"] == [plugin.ref, base_plugin.ref, command.ref] + )) + loads, writes = [], [] + for io_op in io_ops: + used = io_op["prov:used"][0]["@id"] + if "prov:wasDerivedFrom" in prov_doc.shallow_search(lambda node: ("@id" in node and node["@id"] == used)): + writes.append(io_op) + else: + loads.append(io_op) + print(" - Loaded data from:") + for index, load in enumerate(loads): + source_id = load["prov:used"][0]["@id"] + source = prov_doc.shallow_search(lambda node: ("@id" in node and node["@id"] == [source_id]))[0] + print( + f" - Load {index} at {load['prov:startedAtTime']} took " + f"{load['prov:endedAtTime']-load['prov:startedAtTime']} from:\n" + f" - {source['schema:url']}" + ) + print(" - Written data to:") + for index, write in enumerate(writes): + target = prov_doc.shallow_search(lambda node: ( + "prov:wasGeneratedBy" in node and node["prov:wasGeneratedBy"] == [write.ref] + ))[0] + print( + f" - Load {index} at {write['prov:startedAtTime']} took " + f"{write['prov:endedAtTime']-write['prov:startedAtTime']} from:\n" + f" - {target['schema:url']}" + ) From ccdcacfae557409f533f32d36fc9706249aab957 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Tue, 25 Aug 2026 13:29:29 +0200 Subject: [PATCH 25/41] updated provenance diagram --- .../hermes-prov-diagram/hermes-prov.drawio | 980 +++++++++++++++++- docs/adr/hermes-prov-diagram/hermes-prov.svg | 2 +- 2 files changed, 977 insertions(+), 5 deletions(-) diff --git a/docs/adr/hermes-prov-diagram/hermes-prov.drawio b/docs/adr/hermes-prov-diagram/hermes-prov.drawio index 70b06617..6c27a07e 100644 --- a/docs/adr/hermes-prov-diagram/hermes-prov.drawio +++ b/docs/adr/hermes-prov-diagram/hermes-prov.drawio @@ -1,9 +1,625 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -73,11 +689,11 @@ - - + + - + @@ -4156,6 +4772,362 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/adr/hermes-prov-diagram/hermes-prov.svg b/docs/adr/hermes-prov-diagram/hermes-prov.svg index 56e2a032..aea20661 100644 --- a/docs/adr/hermes-prov-diagram/hermes-prov.svg +++ b/docs/adr/hermes-prov-diagram/hermes-prov.svg @@ -1,4 +1,4 @@ -
wasGeneratedBy
used
used
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
wasAssociatedWith
wasAttributedTo
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasInfluencedBy
used
wasDerivedFrom
actedOnBehalfOf
used
used
used
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
actedOnBehalfOf
wasAttributedTo
wasAssociatedWith
actedOnBehalfOf
wasDerivedFrom
wasInfluencedBy
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasAttributedTo
wasAttributedTo
used
used
wasAssociatedWith
wasGeneratedBy
wasAssociatedWith
wasAttributedTo
wasAssociatedWith
wasInformedBy
wasAttributedTo
wasGeneratedBy
wasInformedBy
wasGeneratedBy
used
used
wasGeneratedBy
wasInformedBy
used
wasDerivedFrom
wasDerivedFrom
actedOnBehalfOf
wasAttributedTo
wasAssociatedWith
used
used
used
used
used
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
used
used
wasAssociatedWith
wasInformedBy
used
used
wasInformedBy
wasGeneratedBy
wasInformedBy
used
wasDerivedFrom
wasGeneratedBy
used
wasInformedBy
wasDerivedFrom
wasGeneratedBy
wasDerivedFrom
used
wasGeneratedBy
wasInformedBy
used
wasGeneratedBy
wasGeneratedBy
used
used
used
used
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasInformedBy
used
used
used
used
used
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
used
wasAssociatedWith
wasInformedBy
used
wasInformedBy
wasGeneratedBy
wasInformedBy
used
wasDerivedFrom
wasGeneratedBy
used
wasInformedBy
wasDerivedFrom
wasGeneratedBy
wasDerivedFrom
used
wasGeneratedBy
wasInformedBy
used
wasGeneratedBy
wasGeneratedBy
used
used
used
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasAttributedTo
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasDerivedFrom
wasAssociatedWith
wasInformedBy
wasAttributedTo
wasGeneratedBy
used
wasDerivedFrom
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasAttributedTo
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
used
wasGeneratedBy
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
actedOnBehalfOf
wasGeneratedBy
wasGeneratedBy
actedOnBehalfOf
wasGeneratedBy
wasAttributedTo
wasAssociatedWith
actedOnBehalfOf
used
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasAttributedTo
used
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasAttributedTo
used
used
used
wasAssociatedWith
used
used
wasAssociatedWith
wasInformedBy
wasInformedBy
wasAttributedTo
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
used
used
used
wasAttributedTo
wasAssociatedWith
wasDerivedFrom
wasGeneratedBy
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
wasAssociatedWith
wasInformedBy
wasInformedBy
wasAssociatedWith
wasAttributedTo
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
used
used
used
wasAttributedTo
wasAssociatedWith
wasDerivedFrom
wasGeneratedBy
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasInformedBy
wasInformedBy
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
actedOnBehalfOf
actedOnBehalfOf
wasAttributedTo
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
used
used
used
wasAttributedTo
wasAssociatedWith
wasGeneratedBy
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
actedOnBehalfOf
harvest plugin

name, version, settings
harvest source

path uri
.hermes/harvest/
{plugin_name}/codemeta.json

text, path uri, date created
harvested metadata

data
software-metadata

date, time
map

end time
write

start time, end time
.hermes/harvest/
{plugin_name}/expanded.json

text, path uri, date created
.hermes/harvest/
{plugin_name}/context.json

text, path uri, date created
Legend
design
meaning
provenance: Agent
provenance: Entity
provenance: Activity
bold text
record those properties always
solid lining
record as detailed as possible
dashed lining
record without many details
grayed out
optional / not always existent
name
properties
name
properties
name
properties
harvest plugin

name, version, settings
harvest source

path uri
.hermes/harvest/
{plugin_name}/codemeta.json

text, path uri, date created
harvested metadata

data
software-metadata

data, time
map

end time
write

start time, end time
.hermes/harvest/
{plugin_name}/expanded.json

text, path uri, date created
HARVEST
hermes

version
HERMES cache
load

func, args, kwargs, source, time
harvest base plugin

settings
load

func, args, kwargs, source, time
harvest command

settings
.hermes/harvest/
{plugin_name}/context.json

text, path uri, date created
process plugin

name, version, settings
merge strategies

strategies, time
process plugin

name, version, settings
.hermes/process/result/
codemeta.json

text, path uri, date created
merge strategies

strategies, time
merge strategies

start time, end time
write

start time, end time
.hermes/process/result/
expanded.json

text, path uri, date created
PROCESS
generate merge strategies

start time, end time
process base plugin

settings
generate merge strategies

start time, end time
process command

settings
.hermes/process/result/
context.json

text, path uri, date created
process plugin

name, version, settings
generate merge strategies

start time, end time
merge strategies

start time, end time
merged strategies

strategies, time
merged strategies

strategies, time
harvest plugin

name, version, settings
harvest source

path uri
.hermes/harvest/
{plugin_name}/codemeta.json

text, path uri, date created
harvested metadata

data
software-metadata

data, time
map

end time
write

start time, end time
.hermes/harvest/
{plugin_name}/expanded.json

text, path uri, date created
.hermes/harvest/
{plugin_name}/context.json

text, path uri, date created
load

func, args, kwargs, source, time
software-metadata

data, time
load

start time, end time
software-metadata

data, time
load

start time, end time
software-metadata

data, time
load

start time, end time
used
used
wasAssociatedWith
reject/ replace/ ...
value with other value

start time, end time, strategy used
merge value at key

key, strategy used
merge

start time, end time
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
software-metadata

time, data
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
reject/ replace/ ...
value with other value

start time, end time, strategy used
merge value at key

key, strategy used
merge

start time, end time
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
software-metadata

time, data
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
merge strategies

strategies, time
curate command

settings
software-metadata

data, time
load

start time, end time
curate base plugin

settings
curate plugin

name, version, settings
software-metadata

data, time
.hermes/curate/result/
codemeta.json

text, path uri, time created
write

start time, end time
.hermes/curate/result/
expanded.json

text, path uri, time created
.hermes/curate/result/
context.json

text, path uri, time created
CURATE
used
wasDerivedFrom
actedOnBehalfOf
used
used
used
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
actedOnBehalfOf
wasAttributedTo
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
wasDerivedFrom
wasAttributedTo
wasAttributedTo
wasDerivedFrom
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
deposit command

settings
software-metadata

data, time
load

start time, end time
deposit base plugin

settings
deposit plugin

name, version, settings
mapped data for deposit

data, time
.hermes/deposit/
{deposit_plugin}/deposit.json

text, path uri, time created
write

start time, end time
DEPOSIT
map

start time, end time
updated metadata

data, time
.hermes/deposit/
{deposit_plugin}/result.json

text, path uri, time created
write

start time, end time
 \ No newline at end of file +
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAssociatedWith
used
used
wasGeneratedBy
wasAttributedTo
actedOnBehalfOf
wasDerivedFrom
wasInfluencedBy
wasDerivedFrom
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
used
wasDerivedFrom
wasInfluencedBy
wasDerivedFrom
wasGeneratedBy
wasAssociatedWith
used
wasDerivedFrom
wasGeneratedBy
wasAssociatedWith
used
wasDerivedFrom
wasGeneratedBy
wasAttributedTo
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasAttributedTo
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasAttributedTo
used
wasDerivedFrom
wasGeneratedBy
used
wasDerivedFrom
wasGeneratedBy
wasAssociatedWith
used
wasDerivedFrom
wasGeneratedBy
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasAssociatedWith
wasAssociatedWith
wasInfluencedBy
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasGeneratedBy
used
used
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
wasAssociatedWith
wasAttributedTo
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasInfluencedBy
used
wasDerivedFrom
actedOnBehalfOf
used
used
used
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
actedOnBehalfOf
wasAttributedTo
wasAssociatedWith
actedOnBehalfOf
wasDerivedFrom
wasInfluencedBy
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasAttributedTo
wasAttributedTo
used
used
wasAssociatedWith
wasGeneratedBy
wasAssociatedWith
wasAttributedTo
wasAssociatedWith
wasInformedBy
wasAttributedTo
wasGeneratedBy
wasInformedBy
wasGeneratedBy
used
used
wasGeneratedBy
wasInformedBy
used
wasDerivedFrom
wasDerivedFrom
actedOnBehalfOf
wasAttributedTo
wasAssociatedWith
used
used
used
used
used
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
used
used
wasAssociatedWith
wasInformedBy
used
used
wasInformedBy
wasGeneratedBy
wasInformedBy
used
wasDerivedFrom
wasGeneratedBy
used
wasInformedBy
wasDerivedFrom
wasGeneratedBy
wasDerivedFrom
used
wasGeneratedBy
wasInformedBy
used
wasGeneratedBy
wasGeneratedBy
used
used
used
used
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasInformedBy
used
used
used
used
used
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
used
wasAssociatedWith
wasInformedBy
used
wasInformedBy
wasGeneratedBy
wasInformedBy
used
wasDerivedFrom
wasGeneratedBy
used
wasInformedBy
wasDerivedFrom
wasGeneratedBy
wasDerivedFrom
used
wasGeneratedBy
wasInformedBy
used
wasGeneratedBy
wasGeneratedBy
used
used
used
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasAttributedTo
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasDerivedFrom
wasAssociatedWith
wasInformedBy
wasAttributedTo
wasGeneratedBy
used
wasDerivedFrom
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasAttributedTo
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
used
wasGeneratedBy
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
actedOnBehalfOf
wasGeneratedBy
wasGeneratedBy
actedOnBehalfOf
wasGeneratedBy
wasAttributedTo
wasAssociatedWith
actedOnBehalfOf
used
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasAttributedTo
used
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasAttributedTo
used
used
used
wasAssociatedWith
used
used
wasAssociatedWith
wasInformedBy
wasInformedBy
wasAttributedTo
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
used
used
used
wasAttributedTo
wasAssociatedWith
wasDerivedFrom
wasGeneratedBy
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
wasAssociatedWith
wasInformedBy
wasInformedBy
wasAssociatedWith
wasAttributedTo
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
used
used
used
wasAttributedTo
wasAssociatedWith
wasDerivedFrom
wasGeneratedBy
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasInformedBy
wasInformedBy
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
actedOnBehalfOf
actedOnBehalfOf
wasAttributedTo
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
used
used
used
wasAttributedTo
wasAssociatedWith
wasGeneratedBy
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
actedOnBehalfOf
harvest plugin

name, version, settings
harvest source

path uri
.hermes/harvest/
{plugin_name}/codemeta.json

text, path uri, date created
harvested metadata

data
software-metadata

date, time
map

end time
write

start time, end time
.hermes/harvest/
{plugin_name}/expanded.json

text, path uri, date created
.hermes/harvest/
{plugin_name}/context.json

text, path uri, date created
Legend
design
meaning
provenance: Agent
provenance: Entity
provenance: Activity
bold text
record those properties always
solid lining
record as detailed as possible
dashed lining
record without many details
grayed out
optional / not always existent
name
properties
name
properties
name
properties
harvest plugin

name, version, settings
harvest source

path uri
.hermes/harvest/
{plugin_name}/codemeta.json

text, path uri, date created
harvested metadata

data
software-metadata

data, time
map

end time
write

start time, end time
.hermes/harvest/
{plugin_name}/expanded.json

text, path uri, date created
HARVEST
hermes

version
HERMES cache
load

func, args, kwargs, source, time
harvest base plugin

settings
load

func, args, kwargs, source, time
harvest command

settings
.hermes/harvest/
{plugin_name}/context.json

text, path uri, date created
process plugin

name, version, settings
merge strategies

strategies, time
process plugin

name, version, settings
.hermes/process/result/
codemeta.json

text, path uri, date created
merge strategies

strategies, time
merge strategies

start time, end time
write

start time, end time
.hermes/process/result/
expanded.json

text, path uri, date created
PROCESS
generate merge strategies

start time, end time
process base plugin

settings
generate merge strategies

start time, end time
process command

settings
.hermes/process/result/
context.json

text, path uri, date created
process plugin

name, version, settings
generate merge strategies

start time, end time
merge strategies

start time, end time
merged strategies

strategies, time
merged strategies

strategies, time
harvest plugin

name, version, settings
harvest source

path uri
.hermes/harvest/
{plugin_name}/codemeta.json

text, path uri, date created
harvested metadata

data
software-metadata

data, time
map

end time
write

start time, end time
.hermes/harvest/
{plugin_name}/expanded.json

text, path uri, date created
.hermes/harvest/
{plugin_name}/context.json

text, path uri, date created
load

func, args, kwargs, source, time
software-metadata

data, time
load

start time, end time
software-metadata

data, time
load

start time, end time
software-metadata

data, time
load

start time, end time
used
used
wasAssociatedWith
reject/ replace/ ...
value with other value

start time, end time, strategy used
merge value at key

key, strategy used
merge

start time, end time
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
software-metadata

time, data
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
reject/ replace/ ...
value with other value

start time, end time, strategy used
merge value at key

key, strategy used
merge

start time, end time
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
software-metadata

time, data
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
merge strategies

strategies, time
curate command

settings
software-metadata

data, time
load

start time, end time
curate base plugin

settings
curate plugin

name, version, settings
software-metadata

data, time
.hermes/curate/result/
codemeta.json

text, path uri, time created
write

start time, end time
.hermes/curate/result/
expanded.json

text, path uri, time created
.hermes/curate/result/
context.json

text, path uri, time created
CURATE
used
wasDerivedFrom
actedOnBehalfOf
used
used
used
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
actedOnBehalfOf
wasAttributedTo
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
wasDerivedFrom
wasAttributedTo
wasAttributedTo
wasDerivedFrom
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
deposit command

settings
software-metadata

data, time
load

start time, end time
deposit base plugin

settings
deposit plugin

name, version, settings
mapped data for deposit

data, time
.hermes/deposit/
{deposit_plugin}/deposit.json

text, path uri, time created
write

start time, end time
DEPOSIT
map

start time, end time
updated metadata

data, time
.hermes/deposit/
{deposit_plugin}/result.json

text, path uri, time created
write

start time, end time
used
wasDerivedFrom
actedOnBehalfOf
used
wasGeneratedBy
actedOnBehalfOf
wasAttributedTo
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
wasDerivedFrom
wasInfluencedBy
wasDerivedFrom
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
postprocess command

settings
deposit result

data, time
load

start time, end time
postprocess base plugin

settings
postprocess plugin

name, version, settings
processed data

data, time
some data

text, path uri, time created
write

start time, end time
POSTPROCESS
wasAttributedTo
processed data

data, time
some data

text, path uri, time created
write

start time, end time
loaded data

data, time
some data

path uri
load
start time, end time
loaded data

data, time
some data

path uri
load
start time, end time
wasAttributedTo
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
deposit result

data, time
load

start time, end time
postprocess plugin

name, version, settings
processed data

data, time
some data

text, path uri, time created
write

start time, end time
wasAttributedTo
processed data

data, time
some data

text, path uri, time created
write

start time, end time
loaded data

data, time
some data

path uri
load
start time, end time
loaded data

data, time
some data

path uri
load
start time, end time
wasAttributedTo
 \ No newline at end of file From 3b34b72f5b8b3454d483e96b7fdc95a02f45e4ee Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Tue, 25 Aug 2026 15:02:57 +0200 Subject: [PATCH 26/41] commented ld_prov --- src/hermes/model/provenance/ld_prov.py | 247 ++++++++++++++++++++++--- 1 file changed, 223 insertions(+), 24 deletions(-) diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index 3d5d89f8..eda1bbc1 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -5,25 +5,38 @@ # SPDX-FileContributor: Michael Fritzsche from importlib.metadata import metadata -from typing import Optional, Union +from typing import Any, Callable, Optional, Union from typing_extensions import Self from hermes import utils from hermes.commands.base import HermesCommand, HermesPlugin from hermes.model.types import ld_dict, ld_list -from hermes.model.types.ld_container import EXPANDED_JSON_LD_VALUE, JSON_LD_CONTEXT_DICT +from hermes.model.types.ld_container import EXPANDED_JSON_LD_VALUE, JSON_LD_CONTEXT_DICT, JSON_LD_VALUE from hermes.model.types.ld_context import ALL_CONTEXTS, iri_map class ld_prov_list(ld_list): - NODE_IRI_FORMAT = "_:{type}/{index}" - HERMES_ID = f"https://doi.org/{utils.hermes_doi}" - HERMES_CACHE_ID = "_:hermes/cache" - HERMES_COMMAND_ID_FORMAT = "_:hermes/command/{step}" - HERMES_PLUGIN_ID_FORMAT = "_:hermes/plugin/{step}/{name}" - HERMES_BASE_PLUGIN_ID_FORMAT = "_:hermes/base_plugin/{step}" - PROV_DOC_IRI = iri_map['hermes-rt', "graph"] - INDICES = {} + """ + ld_list with special features for internal provenance collection. + + Attributes: + NODE_IRI_FORMAT (str): (class attribute) The id format of normal nodes. + HERMES_ID (str): (class attribute) The id of the hermes agent. + HERMES_CACHE_ID (str): (class attribute) The id of the hermes cache. + HERMES_COMMAND_ID_FORMAT (str): (class attribute) The id format of hermes commands. + HERMES_PLUGIN_ID_FORMAT (str): (class attribute) The id format of hermes plugins. + HERMES_BASE_PLUGIN_ID_FORMAT (str): (class attribute) The id format of hermes base plugins. + PROV_DOC_IRI (str): (class attribute) The JSON-LD type of the prov_doc itself. + INDICES (dict[str, int]): (class attribute) The counters of the different types of nodes. + """ + NODE_IRI_FORMAT: str = "_:{type}/{index}" + HERMES_ID: str = f"https://doi.org/{utils.hermes_doi}" + HERMES_CACHE_ID: str = "_:hermes/cache" + HERMES_COMMAND_ID_FORMAT: str = "_:hermes/command/{step}" + HERMES_PLUGIN_ID_FORMAT: str = "_:hermes/plugin/{step}/{name}" + HERMES_BASE_PLUGIN_ID_FORMAT: str = "_:hermes/base_plugin/{step}" + PROV_DOC_IRI: str = iri_map['hermes-rt', "graph"] + INDICES: dict[str, int] = {} def __init__( self: Self, @@ -34,15 +47,45 @@ def __init__( index: Optional[int] = None, context: Optional[list[Union[str, JSON_LD_CONTEXT_DICT]]] = ALL_CONTEXTS ) -> None: + """ + Create a new instance of an ld_prov_list, should not be used. + Use :meth:`ld_prov_list.load_ld_prov_list` instead. + See also :meth:`ld_list.__init__`. + + Args: + data (EXPANDED_JSON_LD_VALUE): The expanded json-ld data that represents the list, default is an empty graph + parent (ld_dict | ld_list | None): parent node of this container. + key (str | None): key into the parent container. + index (int | None): index into the parent container. + context (list[str | JSON_LD_CONTEXT_DICT] | None): local context for this container. + + Returns: + None: + """ super().__init__(data, parent=parent, key=key, index=index, context=context) @classmethod - def load_ld_prov_list(cls, data) -> "ld_prov_list": + def load_ld_prov_list(cls: type[Self], data: EXPANDED_JSON_LD_VALUE) -> "ld_prov_list": + """ + Create a new instance of an ld_merge_dict. See also :meth:`ld_dict.__init__`. + + Args: + data (EXPANDED_JSON_LD_VALUE): The expanded json-ld data from which an ld_prov_list is restored. + + Returns: + ld_prov_list: The ld_prov_list loaded from the provided data. + + Raises: + RuntimeError: If an ld_prov_list has/ had been loaded before. + """ + # check if an ld_prov_list has/ had been loaded before if cls.INDICES != {}: raise RuntimeError("Only zero or one objects of class 'ld_prov_list' may exist at every point in time.") + # create ld_prov_list from the data prov_list = cls.from_list( data[0]["@graph"], key=cls.PROV_DOC_IRI, context=ALL_CONTEXTS, container_type="@graph" ) + # initialize counters for different node types for item in prov_list: if not ("@id" in item and item["@id"].startswith("_:")): continue @@ -53,46 +96,112 @@ def load_ld_prov_list(cls, data) -> "ld_prov_list": cls.INDICES[item_id[0]] = int(item_id[1]) return prov_list - def next_node_iri(self, type) -> str: + def next_node_iri(self: Self, type: str) -> str: + """ + Create an iri for a new node of the given type. + + Args: + type (str): The type of the new node + + Returns: + str: The generated iri. + """ + # update counter for the given type if type not in ld_prov_list.INDICES: ld_prov_list.INDICES[type] = 0 ld_prov_list.INDICES[type] += 1 + # generate and return the iri return self.NODE_IRI_FORMAT.format(type=type, index=ld_prov_list.INDICES[type]) - def add_activity(self, *, data={}) -> ld_dict: + def add_activity(self: Self, *, data: JSON_LD_VALUE = {}) -> ld_dict: + """ + Add a new provenance activity to the ld_prov_list using the provided additional data. + + Hint: If no id was specified, one will be generated. Additionaly the types 'prov:Activity' and + 'schema:Action' will be added. + + Args: + data (JSON_LD_VALUE): The additional data for the activity. + + Returns: + ld_dict: The provenance activity as an ld_dict (can be used to update the data in the ld_prov_list). + """ + # add and get the object self.append(data) activity = self[-1] + # add the additional types if "@type" not in data: activity["@type"] = ["prov:Activity", "schema:Action"] else: activity["@type"].extend(["prov:Activity", "schema:Action"]) + # add an id if necessary if "@id" not in data: activity["@id"] = self.next_node_iri("Activity") + # return the object return activity - def add_agent(self, *, data={}) -> ld_dict: + def add_agent(self: Self, *, data: JSON_LD_VALUE = {}) -> ld_dict: + """ + Add a new provenance agent to the ld_prov_list using the provided additional data. + + Hint: If no id was specified, one will be generated. Additionaly the types 'prov:Agent' and + 'schema:SoftwareApplication' will be added. + + Args: + data (JSON_LD_VALUE): The additional data for the agent. + + Returns: + ld_dict: The provenance agent as an ld_dict (can be used to update the data in the ld_prov_list). + """ + # add and get the object self.append(data) agent = self[-1] + # add the additional types if "@type" not in data: agent["@type"] = ["prov:Agent", "schema:SoftwareApplication"] else: agent["@type"].extend(["prov:Agent", "schema:SoftwareApplication"]) + # add an id if necessary if "@id" not in data: agent["@id"] = self.next_node_iri("Agent") + # return the object return agent - def add_entity(self, *, data={}) -> ld_dict: + def add_entity(self: Self, *, data: JSON_LD_VALUE = {}) -> ld_dict: + """ + Add a new provenance entity to the ld_prov_list using the provided additional data. + + Hint: If no id was specified, one will be generated. Additionaly the types 'prov:Entity' and + 'schema:Thing' will be added. + + Args: + data (JSON_LD_VALUE): The additional data for the entity. + + Returns: + ld_dict: The provenance entity as an ld_dict (can be used to update the data in the ld_prov_list). + """ + # add and get the object self.append(data) entity = self[-1] + # add the additional types if "@type" not in data: entity["@type"] = ["prov:Entity", "schema:Thing"] else: entity["@type"].extend(["prov:Entity", "schema:Thing"]) + # add an id if necessary if "@id" not in data: entity["@id"] = self.next_node_iri("Entity") + # return the object return entity - def init_hermes_agents(self) -> None: + def init_hermes_agents(self: Self) -> None: + """ + Initialize the hermes agents for provenance collection. + + Returns: + None: + """ + # add an agent for both hermes itself and the hermes cache hermes = self.add_agent(data={ "@id": ld_prov_list.HERMES_ID, "@type": "schema:SoftwareApplication", @@ -107,6 +216,7 @@ def init_hermes_agents(self) -> None: "schema:version": utils.hermes_version, "prov:actedOnBehalfOf": hermes.ref }) + # add the agents for each command and base plugin for step in ["harvest", "process", "curate", "deposit", "postprocess"]: command = self.add_agent(data={ "@id": ld_prov_list.HERMES_COMMAND_ID_FORMAT.format(step=step), @@ -123,7 +233,17 @@ def init_hermes_agents(self) -> None: "prov:actedOnBehalfOf": command.ref }) - def add_hermes_settings(self, command: HermesCommand) -> None: + def add_hermes_settings(self: Self, command: HermesCommand) -> None: + """ + Add general settings of a hermes command run from the command object. + + Args: + command (HermesCommand): The command object containing information on the run. + + Returns: + None: + """ + # add basic settings hermes = self.get_hermes() hermes.emplace("schema:supportingData") hermes["schema:supportingData"].append({ @@ -148,6 +268,7 @@ def add_hermes_settings(self, command: HermesCommand) -> None: ], "schema:description": f"options for run {len(hermes['schema:supportingData']) + 1} of some hermes step" }) + # add command specific settings for name, values in command.root_settings.model_dump(mode="json").items(): if not isinstance(values, list): values = [values] @@ -164,7 +285,19 @@ def add_hermes_settings(self, command: HermesCommand) -> None: "schema:description": "setting loaded from the config file" }) - def add_settings_to_command(self, step: str, command: HermesCommand) -> None: + def add_settings_to_command(self: Self, step: str, command: HermesCommand) -> None: + """ + Add settings specific to the ran command from the command object. + :meth:`ld_prov_list.add_hermes_settings` must be run before this function. + + Args: + step (str): The step of the settings should be recorded for. + command (HermesCommand): The command object containing information on the run. + + Returns: + None: + """ + # add basics command_prov = self.get_hermes_command(step) command_prov.emplace("schema:supportingData") command_prov["schema:supportingData"].append({ @@ -173,6 +306,7 @@ def add_settings_to_command(self, step: str, command: HermesCommand) -> None: "schema:description": f"options for run {len(command_prov['schema:supportingData']) + 1} of step {step} out" f" of {len(self.get_hermes()['schema:supportingData'])} runs of some hermes step" }) # Needs add_hermes_settings to be called before add_settings_to_command is called! + # add specific settings to the command for name, values in command.settings.model_dump(mode="json").items(): if not isinstance(values, list): values = [values] @@ -188,7 +322,20 @@ def add_settings_to_command(self, step: str, command: HermesCommand) -> None: ] }) - def add_hermes_plugin(self, step: str, name: str, plugin: HermesPlugin, command: HermesCommand) -> ld_dict: + def add_hermes_plugin(self: Self, step: str, name: str, plugin: HermesPlugin, command: HermesCommand) -> ld_dict: + """ + Add a new hermes plugin to the ld_prov_list using the provided additional data. + + Args: + step (str): The step of the plugin. + name (str): The name of the plugin. + plugin (HermesPlugin): The object of the plugin that will be executed. + command (HermesCommand): The command object containing information on the run. + + Returns: + ld_dict: The provenance entity of the plugin (can be used to update the data in the ld_prov_list). + """ + # construct basic data dict data = { "@id": ld_prov_list.HERMES_PLUGIN_ID_FORMAT.format(step=step, name=name), "@type": "schema:SoftwareApplication", @@ -200,6 +347,7 @@ def add_hermes_plugin(self, step: str, name: str, plugin: HermesPlugin, command: }, "prov:actedOnBehalfOf": self.get_hermes_base_plugin(step).ref } + # try adding the settings for the plugin try: for name, values in getattr(command.settings, name).model_dump(mode="json").items(): if not isinstance(values, list): @@ -217,28 +365,70 @@ def add_hermes_plugin(self, step: str, name: str, plugin: HermesPlugin, command: }) except Exception: del data["schema:supportingData"] + # try adding the version of the package of the plugin try: data["schema:softwareVersion"] = metadata(plugin.__module__.split(".")[0])["version"] except Exception: pass + # add the plugin to the ld_prov_list and return the object node = self.add_agent(data=data) return node - def shallow_search(self, query) -> list[ld_dict]: + def shallow_search(self: Self, query: Callable[[ld_dict], Any]) -> list[ld_dict]: + """ + Search the objects in the ld_prov_list for objects for which the query evaluates to True. + + Args: + query (Callable[[ld_dict], Any]): The query used for evaluating the objects. + + Returns: + list[ld_dict]: The objects in the ld_prov_list for which `query` evalutes to True. + """ return [item for item in self if query(item)] - def get_hermes(self) -> ld_dict: + def get_hermes(self: Self) -> ld_dict: + """ + Returns the hermes agent in the ld_prov_list. + + Returns: + ld_dict: The object representing the hermes agent. + """ return self.shallow_search(lambda node: ("@id" in node and node["@id"] == ld_prov_list.HERMES_ID))[0] - def get_hermes_cache(self) -> ld_dict: + def get_hermes_cache(self: Self) -> ld_dict: + """ + Returns the hermes cache agent in the ld_prov_list. + + Returns: + ld_dict: The object representing the hermes cache agent. + """ return self.shallow_search(lambda node: ("@id" in node and node["@id"] == ld_prov_list.HERMES_CACHE_ID))[0] - def get_hermes_base_plugin(self, step) -> ld_dict: + def get_hermes_base_plugin(self: Self, step: str) -> ld_dict: + """ + Returns the base plugin agent in the ld_prov_list of the given step. + + Args: + step (str): The step of which the base plugin agent should be returned. + + Returns: + ld_dict: The object representing the base plugin agent of the given step. + """ return self.shallow_search(lambda node: ( "@id" in node and node["@id"] == ld_prov_list.HERMES_BASE_PLUGIN_ID_FORMAT.format(step=step) ))[0] - def get_hermes_plugin(self, step, name) -> Union[ld_dict, None]: + def get_hermes_plugin(self: Self, step: str, name: str) -> Union[ld_dict, None]: + """ + Returns the plugin agent in the ld_prov_list of the given step with the given name. + + Args: + step (str): The step of which the plugin agent should be returned. + name (str): The name of the plugin agent that should be returned. + + Returns: + ld_dict | None: The object representing the plugin agent of the given step with the given name. + """ search_result = self.shallow_search(lambda node: ( "@id" in node and node["@id"] == ld_prov_list.HERMES_PLUGIN_ID_FORMAT.format(step=step, name=name) )) @@ -247,6 +437,15 @@ def get_hermes_plugin(self, step, name) -> Union[ld_dict, None]: return None def get_hermes_command(self, step) -> ld_dict: + """ + Returns the hermes command agent in the ld_prov_list of the given step. + + Args: + step (str): The step of which the hermes command agent should be returned. + + Returns: + ld_dict: The object representing the hermes command agent of the given step. + """ return self.shallow_search(lambda node: ( "@id" in node and node["@id"] == ld_prov_list.HERMES_COMMAND_ID_FORMAT.format(step=step) ))[0] From 32ffb39eab0655230fd17e2f1eb9516f7b2ee944 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Wed, 26 Aug 2026 16:34:34 +0200 Subject: [PATCH 27/41] comment base of harvest and process --- .readthedocs.yaml | 2 +- src/hermes/commands/harvest/base.py | 136 ++++++++++++--- src/hermes/commands/process/base.py | 254 +++++++++++++++++++--------- 3 files changed, 288 insertions(+), 104 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 0f629fc6..278720c4 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -5,7 +5,7 @@ version: 2 build: - os: ubuntu-20.04 + os: ubuntu-lts-latest tools: python: "3.10" jobs: diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index 7395e192..acacf1aa 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -9,6 +9,8 @@ import datetime from io import IOBase from pathlib import Path +from typing import Any, Callable, Optional +from typing_extensions import Self from pydantic import BaseModel @@ -22,16 +24,51 @@ class HermesHarvestPlugin(HermesPlugin): """Base plugin that does harvesting. + Attributes: + operations (list[tuple[dict[str, str], dict[str, str], dict[str, str]]]): The information recorded on the + load operations executed by the plugin. + TODO: describe the harvesting process and how this is mapped to this plugin. """ - def __init__(self): - self.io_operations: list[tuple[dict, dict, dict]] = [] + + def __init__(self: Self) -> None: + """ + Create a new instance of a HermesHarvestPlugin. + + Returns: + None: + """ + self.operations: list[tuple[dict[str, str], dict[str, str], dict[str, str]]] = [] super().__init__() - def __call__(self, command: HermesCommand) -> SoftwareMetadata: + def __call__(self: Self, command: HermesCommand) -> SoftwareMetadata: + """ + Execute the hermes harvest plugin `self`. + + Args: + command (HermesCommand): The command being executed. + + Returns: + SoftwareMetadata: The harvested metadata. + """ pass - def load(self, func, source, *args, **kwargs): + def load(self: Self, func: Callable, source: Any, *args: Optional[Any], **kwargs: Optional[Any]) -> Any: + """ + Load some data from some source using some function so that the calls provenance information is recorded. + + `func(source, *args, **kwargs)` will be executed. + + Args: + func (Callable): The function used for loading the requested source. + source (Any): The source the data is to be loaded from. + args (Any | None): Additional positional arguments for the load. + kwargs (Any | None): Additional keyword arguments for the load. + + Returns: + Any: The result of the load operation. + """ + # collect basic metadata source_metadata = {"schema:description": "metadata source"} if isinstance(source, IOBase): source_metadata["schema:url"] = Path(source.name).absolute().as_uri() @@ -42,37 +79,60 @@ def load(self, func, source, *args, **kwargs): source_metadata["schema:url"] = Path(source).absolute().as_uri() except Exception: source_metadata["schema:url"] = source - io_operation = { + operation = { "schema:description": "Load operation called with (" f"{source_metadata['schema:url'] if 'schema:url' in source_metadata else str(source)}" f"{', ' + str(args) if args else ''}{', ' + str(kwargs) if kwargs else ''}).", "schema:name": f"{func.__module__}.{func.__qualname__}" } - io_operation["prov:startedAtTime"] = datetime.datetime.now() + operation["prov:startedAtTime"] = datetime.datetime.now() + # execute the load operation result = func(source, *args, **kwargs) - io_operation["prov:endedAtTime"] = datetime.datetime.now() + # complete metadata collection + operation["prov:endedAtTime"] = datetime.datetime.now() loaded_metadata = {"schema:description": "the loaded data", "schema:text": str(result)} - self.io_operations.append((source_metadata, io_operation, loaded_metadata)) + # store metadata + self.operations.append((source_metadata, operation, loaded_metadata)) + # return result of the load operation return result class HarvestSettings(BaseModel): - """Generic harvesting settings.""" + """ + Generic harvesting settings. + + Attributes: + sources (list[str]): (class attribute) A list of plugins to be executed. + """ sources: list[str] = [] def remove_harvest_plugin_from_prov_doc(prov_doc: ld_prov_list, plugin: str) -> None: + """ + Removes information on the specified harvest plugin from the given provenance document. + + Args: + prov_doc (ld_prov_list): The provenance document the plugins information is to be removed from. + plugin (str): The name of the plugin of which the information is to be removed. + + Returns: + None: + """ + # get the plugin object from the prov_doc plugin = prov_doc.get_hermes_plugin("harvest", plugin) + # If the plugin isn't contained in the prov_doc, return, otherwise fetch related objects if plugin is None: return related = prov_doc.shallow_search(lambda node: ( ("prov:wasAssociatedWith" in node and plugin.ref in node["prov:wasAssociatedWith"]) or ("prov:wasAttributedTo" in node and plugin.ref in node["prov:wasAttributedTo"]) )) + # If no related objects exist, delete only the plugin if len(related) == 0: del prov_doc[plugin.index] return + # Collect remaining related objects ids = [plugin.ref, *(rel.ref for rel in related)] used_entities = [rel["prov:used"][0]["@id"] for rel in related if "prov:used" in rel] related = prov_doc.shallow_search(lambda node: node["@id"] in used_entities) @@ -81,6 +141,7 @@ def remove_harvest_plugin_from_prov_doc(prov_doc: ld_prov_list, plugin: str) -> "wasAssociatedWith", "wasAttributedTo", "wasGeneratedBy", "used", "wasDerivedFrom", "wasInformedBy" ] )) + # delete all collected objects del prov_doc[plugin.index] for item in related: items = prov_doc.shallow_search(lambda node: ("@id" in node and node["@id"] == item["@id"])) @@ -89,14 +150,27 @@ def remove_harvest_plugin_from_prov_doc(prov_doc: ld_prov_list, plugin: str) -> class HermesHarvestCommand(HermesCommand): - """ Harvest metadata from configured sources. """ + """ + Harvest metadata from configured sources. + + Attributes: + command_name (str): (class attribute) The name of the command + settings_class (type): (class attribute) The settings class for general harvest settings. + """ - command_name = "harvest" - settings_class = HarvestSettings + command_name: str = "harvest" + settings_class: type = HarvestSettings - def __call__(self, args: argparse.Namespace) -> None: + def __call__(self: Self, args: argparse.Namespace) -> None: + """ + Execute the hermes command `self`. + + Args: + args (Namespace): The arguments of the command. + """ self.args = args self.log.info("# Load provenance from old harvest or create new document.") + # initialize the provenance document for this run prov_doc = self.init_provenance_document() base_plugin = prov_doc.get_hermes_base_plugin("harvest") prov_doc.add_hermes_settings(self) @@ -138,27 +212,31 @@ def __call__(self, args: argparse.Namespace) -> None: stored_at_time = datetime.datetime.now() harvested_any = True + # remove old provenance data from a potential existent old run of this plugin remove_harvest_plugin_from_prov_doc(prov_doc, plugin_name) + # add the plugins provenance information plugin = prov_doc.add_hermes_plugin("harvest", plugin_name, plugin_func, self) - plugin_io_operations = plugin_func.io_operations - outputs = [] - io_ops = [] - for plugin_io_operation in plugin_io_operations: - loaded_source = prov_doc.add_entity(data=plugin_io_operation[0]) - plugin_io_operation[1].update( + # add the collected information on the load operations of the plugin to the provenance document + plugin_operations = plugin_func.operations + outputs, io_ops = [], [] + for plugin_operation in plugin_operations: + loaded_source = prov_doc.add_entity(data=plugin_operation[0]) + plugin_operation[1].update( {"prov:wasAssociatedWith": [base_plugin.ref, plugin.ref], "prov:used": loaded_source.ref} ) - io_op = prov_doc.add_activity(data=plugin_io_operation[1]) - plugin_io_operation[2].update({ + io_op = prov_doc.add_activity(data=plugin_operation[1]) + plugin_operation[2].update({ "prov:wasAttributedTo": plugin.ref, "prov:wasDerivedFrom": loaded_source.ref, "prov:wasGeneratedBy": io_op.ref }) - loaded_data = prov_doc.add_entity(data=plugin_io_operation[2]) + loaded_data = prov_doc.add_entity(data=plugin_operation[2]) + # store references to the added objects outputs.append(loaded_data.ref) io_ops.append(io_op.ref) + # add provenance information on the mapping and returned data map_activity = prov_doc.add_activity(data={ "schema:description": "Maps the loaded data to the JSON-LD contexts vocabulary.", "prov:wasInformedBy": io_ops, @@ -176,6 +254,7 @@ def __call__(self, args: argparse.Namespace) -> None: "prov:generatedAtTime": returned_at_time }) + # add provenance information on the write and stored data write = prov_doc.add_activity(data={ "schema:description": "Writes the harvested metadata into the HERMES cache.", "prov:wasAssociatedWith": [ @@ -222,6 +301,7 @@ def __call__(self, args: argparse.Namespace) -> None: "prov:generatedAtTime": stored_at_time }) + # store provenance information with ctx["provenance"] as cache: cache["result"] = prov_doc.ld_value @@ -231,7 +311,14 @@ def __call__(self, args: argparse.Namespace) -> None: raise HermesPluginRunError("No harvest plugin ran successfully.") @classmethod - def init_provenance_document(cls) -> ld_prov_list: + def init_provenance_document(cls: type[Self]) -> ld_prov_list: + """ + Loads or creates a provenance document. + + Returns: + ld_prov_list: The loaded or created provenance document. + """ + # try loading the document ctx = HermesContext() ctx.prepare_step("harvest") with ctx["provenance"] as cache: @@ -239,6 +326,9 @@ def init_provenance_document(cls) -> ld_prov_list: return ld_prov_list.load_ld_prov_list(cache["result"]) except KeyError: pass + finally: + ctx.finalize_step("harvest") + # initialize a new ld_prov_list because load failed prov_doc = ld_prov_list() prov_doc.init_hermes_agents() return prov_doc diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index 5d3280aa..38e7368d 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -7,6 +7,7 @@ import argparse import datetime from typing import Optional +from typing_extensions import Self from pydantic import BaseModel @@ -18,31 +19,54 @@ from hermes.model.merge.action import MergeAction from hermes.model.merge.container import ld_merge_dict from hermes.model.provenance.ld_prov import ld_prov_list +from hermes.model.types import ld_dict class HermesProcessPlugin(HermesPlugin): - """ Base plugin that defines additional merge strategies.""" + """ Base plugin that defines additional merge strategies. """ - def __call__(self, command: HermesCommand) -> dict[Optional[str], dict[Optional[str], MergeAction]]: + def __call__(self: Self, command: HermesCommand) -> dict[Optional[str], dict[Optional[str], MergeAction]]: + """ + Execute the hermes process plugin `self`. + + Args: + command (HermesCommand): The command being executed. + + Returns: + dict[str | None, dict[str | None, MergeAction]]: The merge strategies. + """ pass class ProcessSettings(BaseModel): - """Generic deposition settings.""" + """ + Generic deposition settings. - sources: list = [] - plugins: list = ["codemeta"] + Attributes: + sources (list[str]): (class attribute) A list of harvest plugins whoose results should be processed. + plugins (list[str]): (class attribute) A list of plugins to be executed. + """ + + sources: list[str] = [] + plugins: list[str] = ["codemeta"] class HermesProcessCommand(HermesCommand): - """ Process the collected metadata into a common dataset. """ + """ + Process the collected metadata into a common dataset. + + Attributes: + command_name (str): (class attribute) The name of the command + settings_class (type): (class attribute) The settings class for general process settings. + """ - command_name = "process" - settings_class = ProcessSettings + command_name: str = "process" + settings_class: type = ProcessSettings def __call__(self, args: argparse.Namespace) -> None: self.args = args self.log.info("# Load provenance data from harvest step") + # try loading and adding general information to the provenance document prov_doc = self.load_prov_doc() if prov_doc is not None: prov_doc.add_hermes_settings(self) @@ -51,7 +75,7 @@ def __call__(self, args: argparse.Namespace) -> None: hermes_cache = prov_doc.get_hermes_cache() self.log.info("# Metadata processing") - merged_doc = ld_merge_dict([{}], prov_doc) + merge_doc = ld_merge_dict([{}], prov_doc) if not self.settings.plugins: self.log.critical( @@ -60,15 +84,103 @@ def __call__(self, args: argparse.Namespace) -> None: ) raise MisconfigurationError("Explicit configuration to use no process plugin.") - # Get all harvesters + # Get all harvesters whoose results should be merged harvester_names = self.settings.sources if self.settings.sources else self.root_settings.harvest.sources if not harvester_names: self.log.critical("# No harvesters to merge from were configured.") raise MisconfigurationError("No harvesters to merge from were configured.") + # generate strategies and add them to the merge_doc + # add provenance information on the process to the prov_doc if provenance is recorded + strategy_action, merged_strategies = self.add_strategies_to_merge_doc(merge_doc, prov_doc) + + # load data and merge it + # add provenance information on the process to the prov_doc if provenance is recorded + last_action, last_data = self.merge_data_from_harvesters( + merge_doc, harvester_names, prov_doc, strategy_action, merged_strategies + ) + + ctx = HermesContext() + self.log.info("## Store processed metadata") + # store processed data + ctx.prepare_step("process") + begin_store_at_time = datetime.datetime.now() + with ctx["result"] as result_ctx: + result_ctx["codemeta"] = merge_doc.compact() + result_ctx["context"] = {"@context": merge_doc.full_context} + result_ctx["expanded"] = merge_doc.ld_value + stored_at_time = datetime.datetime.now() + + if prov_doc is not None: + # add provenance information on the write and stored data + write = prov_doc.add_activity(data={ + "schema:description": "Writes the processed metadata into the HERMES cache.", + "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref], + "prov:used": last_data.ref, + "prov:wasInformedBy": last_action.ref, + "prov:startedAtTime": begin_store_at_time, + "prov:endedAtTime": stored_at_time + }) + prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The compacted version of the processed metadata.", + "schema:text": str(merge_doc.compact()), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "process" / "result" / "codemeta.json").absolute().as_uri(), + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": last_data.ref, + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": stored_at_time + }) + prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The context of the processed metadata.", + "schema:text": str({"@context": merge_doc.full_context}), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "process" / "result" / "context.json").absolute().as_uri(), + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": last_data.ref, + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": stored_at_time + }) + prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The expanded version of the processed metadata.", + "schema:text": str(merge_doc.ld_value), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "process" / "result" / "expanded.json").absolute().as_uri(), + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": last_data.ref, + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": stored_at_time + }) + + # store provenance data + with ctx["provenance"] as cache: + cache["result"] = prov_doc.ld_value + + ctx.finalize_step("process") + + def add_strategies_to_merge_doc( + self: Self, merge_doc: ld_merge_dict, prov_doc: Optional[ld_prov_list] + ) -> tuple[Optional[ld_dict], Optional[ld_dict]]: + """ + Adds strategies to the merge doc that are generated by the process plugins and add provenance information to the + prov_doc. + + Args: + merge_doc (ld_merge_dict): The merge_doc the strategies are to be added to. + prov_doc (ld_prov_list | None): The provenance document where the information is to be recorded. + + Returns: + tuple[ld_dict | None, ld_dict | None]: The object of the last merge of strategies and the object of the + merged strategies. + """ self.log.info("## Load and run the plugins") any_strategies_loaded = False strategy_action, merged_strategies = None, None + if prov_doc is not None: + process_command = prov_doc.get_hermes_command("process") # add the strategies from the plugins for plugin_name in reversed(self.settings.plugins): self.log.info(f"### Load {plugin_name} plugin") @@ -92,12 +204,13 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info(f"### Add the strategies to the merge document {plugin_name} plugin") # add strategies to the merge document merge_strategies_start = datetime.datetime.now() - merged_doc.add_strategy(additional_strategies) + merge_doc.add_strategy(additional_strategies) merge_strategies_end = datetime.datetime.now() any_strategies_loaded = True if prov_doc is None: continue + # add plugin and information on the generation of the merge strategies to the provenance document plugin = prov_doc.add_hermes_plugin("process", plugin_name, plugin_func, self) new_strategy_generation = prov_doc.add_activity(data={ "schema:description": "generate new merge strategies", @@ -114,9 +227,11 @@ def __call__(self, args: argparse.Namespace) -> None: "prov:generatedAtTime": generate_strategies_end }) if merged_strategies is None: + # only first pass for interiteration connections merged_strategies = new_strategies strategy_action = new_strategy_generation continue + # add information on the merge of the old merge strategies with the new ones strategy_action = prov_doc.add_activity(data={ "schema:description": "merging the new strategies into the others", "prov:used": [merged_strategies.ref, new_strategies.ref], @@ -125,25 +240,53 @@ def __call__(self, args: argparse.Namespace) -> None: "prov:startedAtTime": merge_strategies_start, "prov:endedAtTime": merge_strategies_end }) - merged_strategies = prov_doc.add_entity(data={ # TODO: record strategies + merged_strategies = prov_doc.add_entity(data={ "schema:description": "the merge strategies of multiple plugins merged together", - "schema:text": str(merged_doc.strategies), # TODO: maybe "prov:value" instead? + "schema:text": str(merge_doc.strategies), # TODO: maybe "prov:value" instead? "prov:wasDerivedFrom": [merged_strategies.ref, new_strategies.ref], "prov:wasGeneratedBy": strategy_action.ref, "prov:wasAttributedTo": process_command.ref, "prov:generatedAtTime": merge_strategies_end }) + # error if no strategies could be loaded if not any_strategies_loaded: self.log.critical("## No process plugin was ran successfully.") raise HermesPluginRunError("No process plugin was ran successfully.") - ctx = HermesContext() - ctx.prepare_step('harvest') + return strategy_action, merged_strategies + + def merge_data_from_harvesters( + self: Self, + merge_doc: ld_merge_dict, + harvester_names: list[str], + prov_doc: Optional[ld_prov_list], + strategy_action: Optional[ld_dict], + merged_strategies: Optional[ld_dict] + ) -> tuple[Optional[ld_dict], Optional[ld_dict]]: + """ + Load data from the harvest plugins and merge it and then add provenance information to the prov_doc. + + Args: + merge_doc (ld_merge_dict): The merge_doc the strategies are to be added to. + harvester_names (list[str]): The harvesters whoose data is to be loaded and merged. + prov_doc (ld_prov_list | None): The provenance document where the information is to be recorded. + strategy_action (ld_dict | None): The last action merging strategies (prov object). + merged_strategies (ld_dict | None): The result of the strategy merges (prov object). + + Returns: + tuple[ld_dict | None, ld_dict | None]: The object of the last merge and the object of the merged data. + """ + if prov_doc is not None: + process_command = prov_doc.get_hermes_command("process") + hermes_cache = prov_doc.get_hermes_cache() # merge data from harvesters self.log.info("## Merge the metadata of the harvesters") + ctx = HermesContext() + ctx.prepare_step('harvest') merged_any = False + last_action, last_data = None, None for harvester in harvester_names: self.log.info(f"### Load data from {harvester} plugin") # load data from harvester @@ -189,7 +332,7 @@ def __call__(self, args: argparse.Namespace) -> None: "prov:generatedAtTime": load_end }) if merged_any: - # One pass must have been completed already. + # One pass must have been completed successfully already. new_action = prov_doc.add_activity(data={ "schema:description": "merges the old data object with the new data", "prov:used": [last_data.ref, new_data.ref, merged_strategies.ref], @@ -197,13 +340,13 @@ def __call__(self, args: argparse.Namespace) -> None: "prov:wasAssociatedWith": process_command.ref }) # initial merge action of the merge # set the starting objects of the merge - merged_doc.prov_objects = [new_action, new_data, last_data, merged_strategies, strategy_action] + merge_doc.prov_objects = [new_action, new_data, last_data, merged_strategies, strategy_action] self.log.info(f"### Merge data from {harvester} plugin") # merge data into the merge dict try: merge_start = datetime.datetime.now() - merged_doc.update(metadata) + merge_doc.update(metadata) merge_end = datetime.datetime.now() except Exception as e: # TODO: Maybe this state is recoverable by starting over again and skipping this plugin. @@ -214,8 +357,9 @@ def __call__(self, args: argparse.Namespace) -> None: if merged_any: new_action["prov:startedAtTime"] = merge_start new_action["prov:endedAtTime"] = merge_end - last_action = merged_doc.prov_objects[0] if merged_any else new_action - last_data = merged_doc.prov_objects[2] if merged_any else new_data + # set the last action and last data objects for next iteration + last_action = merge_doc.prov_objects[0] if merged_any else new_action + last_data = merge_doc.prov_objects[2] if merged_any else new_data merged_any = True # error if nothing was merged @@ -223,74 +367,24 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.critical("No metadata has been merged, the loading of the data failed for all harvesters.") raise RuntimeError("No metadata has been merged.") - self.log.info("## Store processed metadata") - # store processed data - ctx.prepare_step("process") - begin_store_at_time = datetime.datetime.now() - with ctx["result"] as result_ctx: - result_ctx["codemeta"] = merged_doc.compact() - result_ctx["context"] = {"@context": merged_doc.full_context} - result_ctx["expanded"] = merged_doc.ld_value - stored_at_time = datetime.datetime.now() - - if prov_doc is not None: - write = prov_doc.add_activity(data={ - "schema:description": "Writes the processed metadata into the HERMES cache.", - "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref], - "prov:used": last_data.ref, - "prov:wasInformedBy": last_action.ref, - "prov:startedAtTime": begin_store_at_time, - "prov:endedAtTime": stored_at_time - }) - # TODO: add more info - prov_doc.add_entity(data={ - "@type": "schema:CreativeWork", - "schema:description": "The compacted version of the processed metadata.", - "schema:text": str(merged_doc.compact()), # TODO: maybe "prov:value" instead? - "schema:encodingFormat": "application/json", - "schema:url": (ctx.cache_dir / "process" / "result" / "codemeta.json").absolute().as_uri(), - "prov:wasGeneratedBy": write.ref, - "prov:wasDerivedFrom": last_data.ref, - "prov:wasAttributedTo": hermes_cache.ref, - "prov:generatedAtTime": stored_at_time - }) - prov_doc.add_entity(data={ - "@type": "schema:CreativeWork", - "schema:description": "The context of the processed metadata.", - "schema:text": str({"@context": merged_doc.full_context}), # TODO: maybe "prov:value" instead? - "schema:encodingFormat": "application/json", - "schema:url": (ctx.cache_dir / "process" / "result" / "context.json").absolute().as_uri(), - "prov:wasGeneratedBy": write.ref, - "prov:wasDerivedFrom": last_data.ref, - "prov:wasAttributedTo": hermes_cache.ref, - "prov:generatedAtTime": stored_at_time - }) - prov_doc.add_entity(data={ - "@type": "schema:CreativeWork", - "schema:description": "The expanded version of the processed metadata.", - "schema:text": str(merged_doc.ld_value), # TODO: maybe "prov:value" instead? - "schema:encodingFormat": "application/json", - "schema:url": (ctx.cache_dir / "process" / "result" / "expanded.json").absolute().as_uri(), - "prov:wasGeneratedBy": write.ref, - "prov:wasDerivedFrom": last_data.ref, - "prov:wasAttributedTo": hermes_cache.ref, - "prov:generatedAtTime": stored_at_time - }) - - with ctx["provenance"] as cache: - cache["result"] = prov_doc.ld_value - - ctx.finalize_step("process") + return last_action, last_data - ctx.finalize_step("harvest") + def load_prov_doc(self: Self) -> Optional[ld_prov_list]: + """ + Loads the provenance document of the harvest step. - def load_prov_doc(self) -> Optional[ld_prov_list]: + Returns: + ld_prov_list | None: The loaded provenance document or None if the load failed. + """ + # set up HermesContext ctx = HermesContext() ctx.prepare_step("harvest") with ctx["provenance"] as cache: + # try load try: return ld_prov_list.load_ld_prov_list(cache["result"]) except Exception: + # log the warning and return None self.log.warning( "The provenance data from the harvest step could not be loaded. " "Processing will proceed without collecting provenance data.", From 53276819d3cb08bacab7c4df47f0a6bc30959d6b Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Thu, 27 Aug 2026 15:14:42 +0200 Subject: [PATCH 28/41] comment base classes of remaining hermes steps --- src/hermes/commands/curate/base.py | 69 ++++++++- src/hermes/commands/deposit/base.py | 198 +++++++++++++++++++----- src/hermes/commands/harvest/base.py | 17 +- src/hermes/commands/postprocess/base.py | 161 ++++++++++++++++--- src/hermes/commands/process/base.py | 29 +++- 5 files changed, 401 insertions(+), 73 deletions(-) diff --git a/src/hermes/commands/curate/base.py b/src/hermes/commands/curate/base.py index b151e9f0..e7de9a60 100644 --- a/src/hermes/commands/curate/base.py +++ b/src/hermes/commands/curate/base.py @@ -7,6 +7,7 @@ import argparse import datetime from typing import Optional +from typing_extensions import Self from pydantic import BaseModel @@ -21,29 +22,67 @@ class HermesCuratePlugin(HermesPlugin): """ Base plugin for curate plugins. """ - def __call__(self, command: HermesCommand, metadata: SoftwareMetadata) -> SoftwareMetadata: + def __call__(self: Self, command: "HermesCurateCommand", metadata: SoftwareMetadata) -> SoftwareMetadata: + """ + Execute the hermes curate plugin `self`. + + Args: + command (HermesCurateCommand): The command being executed. + metadata (SoftwareMetadata): The metadata to be curated. + + Returns: + SoftwareMetadata: The curated metadata. + """ pass class CurateSettings(BaseModel): - """Generic deposition settings.""" + """ + Generic deposition settings. + + Attributes: + plugin (str): The plugin to be executed. + """ plugin: str = "pass_curate" class HermesCurateCommand(HermesCommand): - """ Curate the unified metadata before deposition. """ + """ + Curate the unified metadata before deposition. + + Attributes: + args (Namespace): The arguments of the command. + command_name (str): (class attribute) The name of the command. + settings_class (type): (class attribute) The settings class for general curate settings. + """ - command_name = "curate" - settings_class = CurateSettings + command_name: str = "curate" + settings_class: type = CurateSettings - def __call__(self, args: argparse.Namespace) -> None: + def __call__(self: Self, args: argparse.Namespace) -> None: + """ + Execute the hermes command `self`. + + Args: + args (Namespace): The arguments of the command. + + Returns: + None: + + Raises: + HermesValidationError: If the results of the process step couldn't be loaded. + MisconfigurationError: If the curation plugin wasn't found. + HermesPluginRunError: If something went wrong in the plugin run. + """ self.args = args self.log.info("# Load provenance data from process step") + # try loading and adding general information to the provenance document prov_doc = self.load_prov_doc() if prov_doc is not None: prov_doc.add_hermes_settings(self) prov_doc.add_settings_to_command("curate", self) + # get basic hermes objects to reference later curate_command = prov_doc.get_hermes_command("curate") curate_base_plugin = prov_doc.get_hermes_base_plugin("curate") process_command = prov_doc.get_hermes_command("process") @@ -52,6 +91,7 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info("# Metadata curation") plugin_name = self.settings.plugin + # set up HermesContext ctx = HermesContext() ctx.prepare_step("curate") @@ -97,7 +137,9 @@ def __call__(self, args: argparse.Namespace) -> None: stored_at_time = datetime.datetime.now() if prov_doc is not None: + # add information on the curate plugin curate_plugin = prov_doc.add_hermes_plugin("curate", plugin_name, plugin_func, self) + # get objects from process store_action_of_process = prov_doc.shallow_search(lambda node: ( "prov:wasAssociatedWith" in node and node["prov:wasAssociatedWith"] == [process_command.ref, hermes_cache.ref] and @@ -106,6 +148,7 @@ def __call__(self, args: argparse.Namespace) -> None: stored_results_of_process = [res.ref for res in prov_doc.shallow_search(lambda node: ( "prov:wasGeneratedBy" in node and node["prov:wasGeneratedBy"] == [store_action_of_process.ref] ))] + # add information on load, loaded data and curated data load_action = prov_doc.add_activity(data={ "schema:description": "loads the data from process step", "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref], @@ -132,6 +175,7 @@ def __call__(self, args: argparse.Namespace) -> None: "prov:wasDerivedFrom": loaded_data.ref, "prov:generatedAtTime": end_curation_time }) + # add provenance information on the write and stored curated metadata write = prov_doc.add_activity(data={ "schema:description": "Writes the processed metadata into the HERMES cache.", "prov:wasAssociatedWith": [curate_command.ref, hermes_cache.ref], @@ -139,7 +183,6 @@ def __call__(self, args: argparse.Namespace) -> None: "prov:startedAtTime": begin_store_at_time, "prov:endedAtTime": stored_at_time }) - # TODO: add more info prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "The compacted version of the processed metadata.", @@ -174,18 +217,28 @@ def __call__(self, args: argparse.Namespace) -> None: "prov:generatedAtTime": stored_at_time }) + # store provenance information with ctx["provenance"] as cache: cache["result"] = prov_doc.ld_value ctx.finalize_step("curate") - def load_prov_doc(self) -> Optional[ld_prov_list]: + def load_prov_doc(self: Self) -> Optional[ld_prov_list]: + """ + Loads the provenance document of the process step. + + Returns: + ld_prov_list | None: The loaded provenance document or None if the load failed. + """ + # set up HermesContext ctx = HermesContext() ctx.prepare_step("process") with ctx["provenance"] as cache: + # try load try: return ld_prov_list.load_ld_prov_list(cache["result"]) except Exception: + # log the warning and return None self.log.warning( "The provenance data from the process step could not be loaded. " "Processing will proceed without collecting provenance data.", diff --git a/src/hermes/commands/deposit/base.py b/src/hermes/commands/deposit/base.py index 0d2c6cdc..ab5e5b62 100644 --- a/src/hermes/commands/deposit/base.py +++ b/src/hermes/commands/deposit/base.py @@ -10,6 +10,7 @@ import argparse import datetime from typing import Optional +from typing_extensions import Self from pydantic import BaseModel @@ -22,36 +23,56 @@ class BaseDepositPlugin(HermesPlugin): - """Base class that implements the generic deposition workflow. + """ + Base class that implements the generic deposition workflow. + + Attributes: + command (HermesCommand): The command running this plugin. + metadata (SoftwareMetadata): The loaded curated metadata. TODO: describe workflow... needs refactoring to be less stateful! """ - def __call__(self, command: HermesCommand, prov_doc: ld_prov_list) -> None: - """Initiate the deposition process. + def __call__(self: Self, command: "HermesDepositCommand", prov_doc: Optional[ld_prov_list]) -> None: + """ + Initiate the deposition process. This calls a list of additional methods on the class, none of which need to be implemented. + + Args: + command (HermesDepositCommand): The command running this plugin. + prov_doc (ld_prov_list | None): The provenance document the provenance information is to be recorded in. + + Returns: + None: + + Raises: + HermesValidationError: If the metadata from the curation step couldn't be loaded. """ self.command = command target = command.settings.target - self.ctx = HermesContext() - self.ctx.prepare_step("deposit") - self.ctx.prepare_step("curate") + # set up HermesContext + ctx = HermesContext() + ctx.prepare_step("curate") + # load curated metadata try: start_of_load = datetime.datetime.now() - self.metadata = SoftwareMetadata.load_from_cache(self.ctx, "result") + self.metadata = SoftwareMetadata.load_from_cache(ctx, "result") end_of_load = datetime.datetime.now() except Exception as e: raise HermesValidationError("The results of the curate step are invalid.") from e - self.ctx.finalize_step("curate") + ctx.finalize_step("curate") if prov_doc is not None: + # add provenance information on the plugin plugin = prov_doc.add_hermes_plugin("deposit", target, self, command) + # get basic hermes objects to reference later deposit_command = prov_doc.get_hermes_command("deposit") curate_command = prov_doc.get_hermes_command("curate") deposit_base_plugin = prov_doc.get_hermes_base_plugin("deposit") hermes_cache = prov_doc.get_hermes_cache() + # get objects from the curate step store_action_curate = prov_doc.shallow_search(lambda node: ( "prov:wasAssociatedWith" in node and node["prov:wasAssociatedWith"] == [curate_command.ref, hermes_cache.ref] and @@ -61,6 +82,7 @@ def __call__(self, command: HermesCommand, prov_doc: ld_prov_list) -> None: results_curate = [item.ref for item in prov_doc.shallow_search(lambda node: ( "prov:wasGeneratedBy" in node and node["prov:wasGeneratedBy"] == [store_action_curate.ref] ))] + # record provenance information on the load action and the loaded data load_action = prov_doc.add_activity(data={ "schema:description": "Loads the results of the curate step.", "prov:used": results_curate, @@ -78,15 +100,18 @@ def __call__(self, command: HermesCommand, prov_doc: ld_prov_list) -> None: "prov:generatedAtTime": end_of_load }) + # prepare, map metadata and store the result self.prepare() start_of_map = datetime.datetime.now() deposit = self.map_metadata() end_of_map = datetime.datetime.now() - with self.ctx[target] as cache: + ctx.prepare_step("deposit") + with ctx[target] as cache: cache["deposit"] = deposit end_of_store = datetime.datetime.now() if prov_doc is not None: + # record provenance information on map, mapped data, store and the stored data map_action = prov_doc.add_activity(data={ "schema:description": "Maps the metadata to the format required by the deposition target.", "prov:used": loaded_data.ref, @@ -115,26 +140,29 @@ def __call__(self, command: HermesCommand, prov_doc: ld_prov_list) -> None: "schema:description": "The stored version of the mapped metadata.", "schema:text": str(deposit), # TODO: maybe "prov:value" instead? "schema:encodingFormat": "application/json", - "schema:url": (self.ctx.cache_dir / "deposit" / target / "deposit.json").absolute().as_uri(), + "schema:url": (ctx.cache_dir / "deposit" / target / "deposit.json").absolute().as_uri(), "prov:wasGeneratedBy": store_mapped_data.ref, "prov:wasDerivedFrom": mapped_data.ref, "prov:wasAttributedTo": hermes_cache.ref, "prov:generatedAtTime": end_of_store }) + # create version if self.is_initial_publication(): self.create_initial_version() else: self.create_new_version() + # update mapped data and store the result updated_deposit = self.update_metadata() end_of_update_map = datetime.datetime.now() - with self.ctx[target] as cache: + with ctx[target] as cache: cache["result"] = updated_deposit end_of_second_store = datetime.datetime.now() - self.ctx.finalize_step("deposit") + ctx.finalize_step("deposit") if prov_doc is not None: + # record update, store and stored data updated_mapped_data = prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "The updated mapped metadata.", @@ -155,98 +183,178 @@ def __call__(self, command: HermesCommand, prov_doc: ld_prov_list) -> None: "schema:description": "The stored version of the updated mapped metadata.", "schema:text": str(updated_deposit), # TODO: maybe "prov:value" instead? "schema:encodingFormat": "application/json", - "schema:url": (self.ctx.cache_dir / "deposit" / target / "result.json").absolute().as_uri(), + "schema:url": (ctx.cache_dir / "deposit" / target / "result.json").absolute().as_uri(), "prov:wasGeneratedBy": store_updated_mapped_data.ref, "prov:wasDerivedFrom": updated_mapped_data.ref, "prov:wasAttributedTo": hermes_cache.ref, "prov:generatedAtTime": end_of_second_store }) + # finish up deposit self.delete_artifacts() self.upload_artifacts() self.publish() - def prepare(self) -> None: - """Prepare the deposition. + def prepare(self: Self) -> None: + """ + Prepare the deposition. This method may be implemented to check whether config and context match some initial conditions. If no exceptions are raised, execution continues. + + Returns: + None: """ pass @abc.abstractmethod - def map_metadata(self) -> dict: - """Map the given metadata to the target schema of the deposition platform and return it. + def map_metadata(self: Self) -> dict: + """ + Map the given metadata to the target schema of the deposition platform and return it. When mapping metadata, make sure to add traces to the HERMES software, e.g. via DataCite's ``relatedIdentifier`` using the ``isCompiledBy`` relation. Ideally, the value of the relation target should be of the respective type for DOIs in your metadata schema, with the value itself being the DOI for the version of the HERMES software you are using. + + Returns: + dict: The mapped metadata. """ pass - def is_initial_publication(self) -> bool: - """Decide whether to do an initial publication or publish a new version. + def is_initial_publication(self: Self) -> bool: + """ + Decide whether to do an initial publication or publish a new version. Returning ``True`` indicates that publication of an initial version will be executed, resulting in a call of :meth:`create_initial_version`. ``False`` indicates a new version of an existing publication, leading to a call of :meth:`create_new_version`. By default, this returns ``True``. + + Returns: + bool: Whether or not it is the initial publication. """ return True - def create_initial_version(self) -> None: - """Create an initial version of the publication on the target platform.""" + def create_initial_version(self: Self) -> None: + """ + Create an initial version of the publication on the target platform. + + Returns: + None: + """ pass - def create_new_version(self) -> None: - """Create a new version of an existing publication on the target platform.""" + def create_new_version(self: Self) -> None: + """ + Create a new version of an existing publication on the target platform. + + Returns: + None: + """ pass @abc.abstractmethod - def update_metadata(self) -> dict: - """Update the metadata of the newly created version and return it even if it hasn't changed.""" + def update_metadata(self: Self) -> dict: + """ + Update the metadata of the newly created version and return it even if it hasn't changed. + + Returns: + dict: The updated metadata. + """ pass - def delete_artifacts(self) -> None: - """Delete any superfluous artifacts taken from the previous version of the publication.""" + def delete_artifacts(self: Self) -> None: + """ + Delete any superfluous artifacts taken from the previous version of the publication. + + Returns: + None: + """ pass - def upload_artifacts(self) -> None: - """Upload new artifacts to the target platform.""" + def upload_artifacts(self: Self) -> None: + """ + Upload new artifacts to the target platform. + + Returns: + None: + """ pass @abc.abstractmethod - def publish(self) -> None: - """Publish the newly created deposit on the target platform.""" + def publish(self: Self) -> None: + """ + Publish the newly created deposit on the target platform. + + Returns: + None: + """ pass class DepositSettings(BaseModel): - """Generic deposition settings.""" + """ + Generic deposition settings. + + Attributes: + target (str): The plugin to be executed. + """ target: str = "" class HermesDepositCommand(HermesCommand): - """ Deposit the curated metadata to repositories. """ + """ + Deposit the curated metadata to repositories. + + Attributes: + args (Namespace): The arguments of the command. + command_name (str): (class attribute) The name of the command. + settings_class (type): (class attribute) The settings class for general deposit settings. + """ command_name = "deposit" settings_class = DepositSettings - def init_command_parser(self, command_parser: argparse.ArgumentParser) -> None: - command_parser.add_argument('--file', '-f', nargs=1, action='append', - help="File that should be part of the deposition.") - command_parser.add_argument('--initial', action='store_true', default=False, - help="Allow initial deposition (i.e., minting a new PID).") + def init_command_parser(self: Self, command_parser: argparse.ArgumentParser) -> None: + """ + Add arguments for deposit command. + + Args: + command_parser (ArgumentParser): The used argument parser. + + Returns: + None: + """ + command_parser.add_argument( + '--file', '-f', nargs=1, action='append', help="File that should be part of the deposition." + ) + command_parser.add_argument( + '--initial', action='store_true', default=False, help="Allow initial deposition (i.e., minting a new PID)." + ) + + def __call__(self: Self, args: argparse.Namespace) -> None: + """ + Execute the hermes command `self`. + + Args: + args (Namespace): The arguments of the command. + + Returns: + None: - def __call__(self, args: argparse.Namespace) -> None: + Raises: + MisconfigurationError: If the deposit plugin wasn't found. + HermesPluginRunError: If something went wrong in the plugin run. + """ self.log.info("# Metadata deposition") self.args = args plugin_name = self.settings.target + # try loading and adding general information to the provenance document prov_doc = self.load_prov_doc() if prov_doc is not None: prov_doc.add_hermes_settings(self) @@ -273,19 +381,29 @@ def __call__(self, args: argparse.Namespace) -> None: if prov_doc is None: return + # store provenance result ctx = HermesContext() ctx.prepare_step("deposit") with ctx["provenance"] as cache: cache["result"] = prov_doc.ld_value ctx.finalize_step("deposit") - def load_prov_doc(self) -> Optional[ld_prov_list]: + def load_prov_doc(self: Self) -> Optional[ld_prov_list]: + """ + Loads the provenance document of the curate step. + + Returns: + ld_prov_list | None: The loaded provenance document or None if the load failed. + """ + # set up HermesContext ctx = HermesContext() ctx.prepare_step("curate") with ctx["provenance"] as cache: + # try load try: return ld_prov_list.load_ld_prov_list(cache["result"]) except Exception: + # log the warning and return None self.log.warning( "The provenance data from the curate step could not be loaded. " "Deposition will proceed without collecting provenance data.", diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index acacf1aa..f1691e46 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -41,12 +41,12 @@ def __init__(self: Self) -> None: self.operations: list[tuple[dict[str, str], dict[str, str], dict[str, str]]] = [] super().__init__() - def __call__(self: Self, command: HermesCommand) -> SoftwareMetadata: + def __call__(self: Self, command: "HermesHarvestCommand") -> SoftwareMetadata: """ Execute the hermes harvest plugin `self`. Args: - command (HermesCommand): The command being executed. + command (HermesHarvestCommand): The command being executed. Returns: SoftwareMetadata: The harvested metadata. @@ -154,6 +154,7 @@ class HermesHarvestCommand(HermesCommand): Harvest metadata from configured sources. Attributes: + args (Namespace): The arguments of the command. command_name (str): (class attribute) The name of the command settings_class (type): (class attribute) The settings class for general harvest settings. """ @@ -167,14 +168,22 @@ def __call__(self: Self, args: argparse.Namespace) -> None: Args: args (Namespace): The arguments of the command. + + Returns: + None: + + Raises: + MisconfigurationError: If no plugin is configured to be run. + HermesPluginRunError: If all plugin runs failed. """ self.args = args self.log.info("# Load provenance from old harvest or create new document.") # initialize the provenance document for this run prov_doc = self.init_provenance_document() - base_plugin = prov_doc.get_hermes_base_plugin("harvest") prov_doc.add_hermes_settings(self) prov_doc.add_settings_to_command("harvest", self) + # get basic hermes object to reference later + base_plugin = prov_doc.get_hermes_base_plugin("harvest") self.log.info("# Metadata harvesting") if len(self.settings.sources) == 0: @@ -191,7 +200,7 @@ def __call__(self: Self, args: argparse.Namespace) -> None: self.log.info(f"### Load {plugin_name} plugin") # load plugin try: - plugin_func = self.plugins[plugin_name]() + plugin_func: HermesHarvestPlugin = self.plugins[plugin_name]() except KeyError: self.log.error(f"### Plugin {plugin_name} not found, skipping it now.") continue diff --git a/src/hermes/commands/postprocess/base.py b/src/hermes/commands/postprocess/base.py index d569a2b7..d960f8b9 100644 --- a/src/hermes/commands/postprocess/base.py +++ b/src/hermes/commands/postprocess/base.py @@ -10,6 +10,7 @@ from io import IOBase from pathlib import Path from typing import Any, Callable, Optional +from typing_extensions import Self from pydantic import BaseModel @@ -17,35 +18,90 @@ from hermes.error import HermesPluginRunError from hermes.model.context_manager import HermesContext from hermes.model.provenance.ld_prov import ld_prov_list +from hermes.model.types import ld_dict class HermesPostprocessPlugin(HermesPlugin): - """ Base plugin for postprocess plugins. """ + """ + Base plugin for postprocess plugins. - def __init__(self): - self.cache_operations: list[tuple[str, dict, dict]] = [] - self.load_operations: list[tuple[dict, dict, dict]] = [] - self.write_operations: list[tuple[dict, dict, dict]] = [] + Attributes: + cache_operations (list[tuple[str, dict[str, str], dict[str, str]]]): The information recorded on the + cache load operations executed by the plugin. + load_operations (list[tuple[dict[str, str], dict[str, str], dict[str, str]]]): The information recorded on the + load operations executed by the plugin. + write_operations (list[tuple[dict[str, str], dict[str, str], dict[str, str]]]): The information recorded on the + write operations executed by the plugin. + """ + + def __init__(self: Self) -> None: + """ + Create a new instance of a HermesPostprocessPlugin. + + Returns: + None: + """ + self.cache_operations: list[tuple[str, dict[str, str], dict[str, str]]] = [] + self.load_operations: list[tuple[dict[str, str], dict[str, str], dict[str, str]]] = [] + self.write_operations: list[tuple[dict[str, str], dict[str, str], dict[str, str]]] = [] super().__init__() - def __call__(self, command: HermesCommand) -> None: + def __call__(self: Self, command: "HermesPostprocessCommand") -> None: + """ + Execute the hermes postprocess plugin `self`. + + Args: + command (HermesPostprocessCommand): The command being executed. + + Returns: + None: + """ pass - def get_deposit_result(self, target: str) -> dict: + def get_deposit_result(self: Self, target: str) -> dict: + """ + Load the result of some deposit plugin from the cache so that the calls provenance information is recorded. + + Args: + target (str): The name of the deposit plugin. + + Returns: + dict: The result of the cache load operation. + """ + # collect basic metadata source_metadata = target[:] load_operation = {"schema:description": f"loads the result of deposit plugin {target}"} ctx = HermesContext() ctx.prepare_step("deposit") load_operation["prov:startedAtTime"] = datetime.datetime.now() + # execute the load operation with ctx[target] as cache: res = cache["result"] + # complete metadata collection load_operation["prov:endedAtTime"] = datetime.datetime.now() ctx.finalize_step("deposit") loaded_data = {"schema:description": "the loaded data", "schema:text": str(res)} + # store metadata self.cache_operations.append((source_metadata, load_operation, loaded_data)) + # return result of the load operation return res - def load(self, func: Callable, source: Any, *args, **kwargs) -> Any: + def load(self: Self, func: Callable, source: Any, *args: Optional[Any], **kwargs: Optional[Any]) -> Any: + """ + Load some data from some source using some function so that the calls provenance information is recorded. + + `func(source, *args, **kwargs)` will be executed. + + Args: + func (Callable): The function used for loading the requested source. + source (Any): The source the data is to be loaded from. + args (Any | None): Additional positional arguments for the load. + kwargs (Any | None): Additional keyword arguments for the load. + + Returns: + Any: The result of the load operation. + """ + # collect basic metadata source_metadata = {"schema:description": "metadata source"} if isinstance(source, IOBase): source_metadata["schema:url"] = Path(source.name).absolute().as_uri() @@ -63,13 +119,35 @@ def load(self, func: Callable, source: Any, *args, **kwargs) -> Any: "schema:name": f"{func.__module__}.{func.__qualname__}" } load_operation["prov:startedAtTime"] = datetime.datetime.now() + # execute the load operation result = func(source, *args, **kwargs) + # complete metadata collection load_operation["prov:endedAtTime"] = datetime.datetime.now() loaded_metadata = {"schema:description": "the loaded data", "schema:text": str(result)} + # store metadata self.load_operations.append((source_metadata, load_operation, loaded_metadata)) + # return result of the load operation return result - def write(self, func: Callable, data: Any, destination: Any, *args, **kwargs) -> Any: + def write( + self: Self, func: Callable, data: Any, destination: Any, *args: Optional[Any], **kwargs: Optional[Any] + ) -> Any: + """ + Write some data from some source using some function so that the calls provenance information is recorded. + + `func(source, *args, **kwargs)` will be executed. + + Args: + func (Callable): The function used for writing the requested source. + data (Any): The data that is to be written. + destination (Any): The source the data is to be written to. + args (Any | None): Additional positional arguments for the write. + kwargs (Any | None): Additional keyword arguments for the write. + + Returns: + Any: The result of the write operation. + """ + # collect basic metadata destination_metadata = {"schema:description": "metadata destination"} if isinstance(destination, IOBase): destination_metadata["schema:url"] = Path(destination.name).absolute().as_uri() @@ -87,33 +165,63 @@ def write(self, func: Callable, data: Any, destination: Any, *args, **kwargs) -> "schema:name": f"{func.__module__}.{func.__qualname__}" } write_operation["prov:startedAtTime"] = datetime.datetime.now() + # execute the write operation result = func(data, destination, *args, **kwargs) + # complete metadata collection write_operation["prov:endedAtTime"] = datetime.datetime.now() written_metadata = {"schema:description": "the written data", "schema:text": str(data)} + # store metadata self.write_operations.append((written_metadata, write_operation, destination_metadata)) + # return result of the write operation return result class PostprocessSettings(BaseModel): - """Generic post-processing settings.""" + """ + Generic post-processing settings. - run: list = [] + Attributes: + run (list[str]): A list of plugins to be executed. + """ + + run: list[str] = [] class HermesPostprocessCommand(HermesCommand): - """Post-process the published metadata after deposition.""" + """ + Post-process the published metadata after deposition. + + Attributes: + args (Namespace): The arguments of the command. + command_name (str): (class attribute) The name of the command. + settings_class (type): (class attribute) The settings class for general deposit settings. + """ - command_name = "postprocess" - settings_class = PostprocessSettings + command_name: str = "postprocess" + settings_class: type = PostprocessSettings - def __call__(self, args: argparse.Namespace) -> None: + def __call__(self: Self, args: argparse.Namespace) -> None: + """ + Execute the hermes command `self`. + + Args: + args (Namespace): The arguments of the command. + + Returns: + None: + + Raises: + HermesPluginRunError: If something went wrong with all plugin runs. + """ self.log.info("# Postprocessing") self.args = args plugin_names = self.settings.run + # try loading and adding general information to the provenance document prov_doc = self.load_prov_doc() if prov_doc is not None: prov_doc.add_hermes_settings(self) prov_doc.add_settings_to_command("postprocess", self) + # get basic hermes objects to reference later hermes_cache = prov_doc.get_hermes_cache() postprocess_command = prov_doc.get_hermes_command("postprocess") postprocess_base_plugin = prov_doc.get_hermes_base_plugin("postprocess") @@ -128,7 +236,7 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info(f"### Load {plugin_name} plugin") # load plugin try: - plugin_func = self.plugins[plugin_name]() + plugin_func: HermesPostprocessPlugin = self.plugins[plugin_name]() except KeyError: self.log.error(f"### Plugin {plugin_name} not found.") continue @@ -146,11 +254,15 @@ def __call__(self, args: argparse.Namespace) -> None: if prov_doc is None: continue + # add information on the postprocess plugin plugin = prov_doc.add_hermes_plugin("postprocess", plugin_name, plugin_func, self) + # add the collected information on the io operations of the plugin to the provenance document cache_loads = plugin_func.cache_operations loads = plugin_func.load_operations writes = plugin_func.write_operations - load_actions, loaded_datas = [], [] + load_actions: list[ld_dict] = [] + loaded_datas: list[ld_dict] = [] + # add cache load operations to the provenance document for cache_load in cache_loads: deposit_plugin = prov_doc.get_hermes_plugin("postprocess", cache_load[0]) updated_metadata = prov_doc.shallow_search(lambda node: ( @@ -172,6 +284,7 @@ def __call__(self, args: argparse.Namespace) -> None: "prov:wasDerivedFrom": updated_metadata.ref, "prov:wasAttributedTo": hermes_cache.ref }) + # add load operations to the provenance document for load in loads: source = prov_doc.add_entity(data=load[0]) load_actions.append(prov_doc.add_activity(data=load[1])) @@ -187,6 +300,7 @@ def __call__(self, args: argparse.Namespace) -> None: }) load_actions = [load_action.ref for load_action in load_actions] loaded_datas = [loaded_data.ref for loaded_data in loaded_datas] + # add write operations to the provenance document for write in writes: data = prov_doc.add_entity(data=write[0]) data.update({"prov:wasDerivedFrom": loaded_datas, "prov:wasInfluencedBy": plugin.ref}) @@ -202,23 +316,34 @@ def __call__(self, args: argparse.Namespace) -> None: }) if prov_doc is not None: + # store provenance data ctx = HermesContext() ctx.prepare_step("postprocess") with ctx["provenance"] as cache: cache["result"] = prov_doc.ld_value ctx.finalize_step("postprocess") + # error out if no plugin ran successfully if not ran_any: self.log.critical("## No postprocess plugin ran successfully.") raise HermesPluginRunError("No postprocess plugin ran successfully.") - def load_prov_doc(self) -> Optional[ld_prov_list]: + def load_prov_doc(self: Self) -> Optional[ld_prov_list]: + """ + Loads the provenance document of the postprocess step. + + Returns: + ld_prov_list | None: The loaded provenance document or None if the load failed. + """ + # set up HermesContext ctx = HermesContext() ctx.prepare_step("deposit") with ctx["provenance"] as cache: + # try load try: return ld_prov_list.load_ld_prov_list(cache["result"]) except Exception: + # log the warning and return None self.log.warning( "The provenance data from the deposit step could not be loaded. " "Postprocessing will proceed without collecting provenance data.", diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index 38e7368d..c474fe65 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -25,12 +25,12 @@ class HermesProcessPlugin(HermesPlugin): """ Base plugin that defines additional merge strategies. """ - def __call__(self: Self, command: HermesCommand) -> dict[Optional[str], dict[Optional[str], MergeAction]]: + def __call__(self: Self, command: "HermesProcessCommand") -> dict[Optional[str], dict[Optional[str], MergeAction]]: """ Execute the hermes process plugin `self`. Args: - command (HermesCommand): The command being executed. + command (HermesProcessCommand): The command being executed. Returns: dict[str | None, dict[str | None, MergeAction]]: The merge strategies. @@ -56,6 +56,7 @@ class HermesProcessCommand(HermesCommand): Process the collected metadata into a common dataset. Attributes: + args (Namespace): The arguments of the command. command_name (str): (class attribute) The name of the command settings_class (type): (class attribute) The settings class for general process settings. """ @@ -63,7 +64,20 @@ class HermesProcessCommand(HermesCommand): command_name: str = "process" settings_class: type = ProcessSettings - def __call__(self, args: argparse.Namespace) -> None: + def __call__(self: Self, args: argparse.Namespace) -> None: + """ + Execute the hermes command `self`. + + Args: + args (Namespace): The arguments of the command. + + Returns: + None: + + Raises: + MisconfigurationError: If it was explicitly configured that no process plugin should be run. + MisconfigurationError: If no harvesters have been configured to be used. + """ self.args = args self.log.info("# Load provenance data from harvest step") # try loading and adding general information to the provenance document @@ -71,6 +85,7 @@ def __call__(self, args: argparse.Namespace) -> None: if prov_doc is not None: prov_doc.add_hermes_settings(self) prov_doc.add_settings_to_command("process", self) + # get basic hermes objects to reference later process_command = prov_doc.get_hermes_command("process") hermes_cache = prov_doc.get_hermes_cache() @@ -100,6 +115,7 @@ def __call__(self, args: argparse.Namespace) -> None: merge_doc, harvester_names, prov_doc, strategy_action, merged_strategies ) + # set up HermesContext ctx = HermesContext() self.log.info("## Store processed metadata") # store processed data @@ -175,6 +191,9 @@ def add_strategies_to_merge_doc( Returns: tuple[ld_dict | None, ld_dict | None]: The object of the last merge of strategies and the object of the merged strategies. + + Raises: + HermesPluginRunError: If all plugin runs failed. """ self.log.info("## Load and run the plugins") any_strategies_loaded = False @@ -276,6 +295,10 @@ def merge_data_from_harvesters( Returns: tuple[ld_dict | None, ld_dict | None]: The object of the last merge and the object of the merged data. + + Raises: + RuntimeError: If a merge failed. + RuntimeError: If data from all harvesters couldn't be loaded. """ if prov_doc is not None: process_command = prov_doc.get_hermes_command("process") From 43e600d01ae8c76f79d1bde57b54cf983ca9dd2b Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Thu, 27 Aug 2026 16:59:12 +0200 Subject: [PATCH 29/41] comment base classes and report command base --- src/hermes/commands/base.py | 192 +++++++++++++++++++----- src/hermes/commands/curate/base.py | 4 +- src/hermes/commands/deposit/base.py | 4 +- src/hermes/commands/harvest/base.py | 4 +- src/hermes/commands/postprocess/base.py | 4 +- src/hermes/commands/process/base.py | 4 +- src/hermes/commands/report/base.py | 107 +++++++++++-- 7 files changed, 262 insertions(+), 57 deletions(-) diff --git a/src/hermes/commands/base.py b/src/hermes/commands/base.py index 12e3c994..8cd1f81d 100644 --- a/src/hermes/commands/base.py +++ b/src/hermes/commands/base.py @@ -9,7 +9,8 @@ import logging import pathlib from importlib import metadata -from typing import Type, Union +from typing import Optional +from typing_extensions import Self import toml from pydantic import BaseModel @@ -17,36 +18,54 @@ class HermesSettings(BaseSettings): - """Root class for HERMES configuration model.""" + """ + Root class for HERMES configuration model. + + Attributes: + model_config (SettingsConfigDict): The settings config dict for the settings of hermes. + logging (dict): ... + """ model_config = SettingsConfigDict(env_file_encoding='utf-8') - logging: dict = {} + logging: dict = {} # FIXME: Is this still used? Even if removed, no tests fail... class HermesCommand(abc.ABC): """Base class for a HERMES workflow command. - :cvar NAME: The name of the sub-command that is defined here. + Attributes: + command_name (str): (class attribute) Only defined here for highlighting, the value of the subclass is is used. + settings_class (type): (class attribute) The settings class for the general hermes command settings """ command_name: str = "" - settings_class: Type = HermesSettings + settings_class: type = HermesSettings - def __init__(self, parser: argparse.ArgumentParser): - """Initialize a new instance of any HERMES command. + def __init__(self: Self, parser: argparse.ArgumentParser) -> None: + """ + Initialize a new instance of any HERMES command. - :param parser: The command line parser used for reading command line arguments. + Args: + parser (ArgumentParser): The command line parser used for reading command line arguments. + + Returns: + None: """ self.parser = parser self.plugins = self.init_plugins() self.settings = None self.log = logging.getLogger(f"hermes.{self.command_name}") - self.errors = [] + self.errors = [] # FIXME: not used, right? - def init_plugins(self): - """Collect and initialize the plugins available for the HERMES command.""" + def init_plugins(self: Self) -> dict[str, type["HermesPlugin"]]: + """ + Collect and initialize the plugins available for the HERMES command. + + Returns: + dict[str, HermesPlugin]: A map mapping the plugin name to the plugin class for the current step. + """ # Collect all entry points for this group (i.e., all valid plug-ins for the step) entry_point_group = f"hermes.{self.command_name}" @@ -65,10 +84,20 @@ def init_plugins(self): return group_plugins @classmethod - def derive_settings_class(cls, setting_types: dict[str, Type]) -> None: - """Build a new Pydantic data model class for configuration. + def derive_settings_class(cls: type[Self], setting_types: dict[str, type["HermesPlugin"]]) -> None: + """ + Build a new Pydantic data model class for configuration. This will create a new class that includes all settings from the plugins available. + + Args: + settings_types (dict[str, type]): The settings classes for the plugins. + + Returns: + None: + + Raises: + ValueError: If the command has no settings. """ if cls.settings_class is not None: @@ -88,10 +117,16 @@ def derive_settings_class(cls, setting_types: dict[str, Type]) -> None: elif setting_types: raise ValueError(f"Command {cls.command_name} has no settings, hence plugin must not have settings, too.") - def init_common_parser(self, parser: argparse.ArgumentParser) -> None: - """Initialize the common command line arguments available for all HERMES sub-commands. + def init_common_parser(self: Self, parser: argparse.ArgumentParser) -> None: + """ + Initialize the common command line arguments available for all HERMES sub-commands. + + Args: + parser (ArgumentsParser): The base command line parser used as entry point when reading command line + arguments. - :param parser: The base command line parser used as entry point when reading command line arguments. + Returns: + None: """ parser.add_argument( @@ -117,26 +152,47 @@ def init_common_parser(self, parser: argparse.ArgumentParser) -> None: "VALUE is the actual value.", ) - def init_command_parser(self, command_parser: argparse.ArgumentParser) -> None: - """Initialize the command line arguments available for this specific HERMES sub-commands. + def init_command_parser(self: Self, command_parser: argparse.ArgumentParser) -> None: + """ + Initialize the command line arguments available for this specific HERMES sub-commands. You should override this method to add your custom arguments to the command line parser of the respective sub-command. - :param command_parser: The command line sub-parser responsible for the HERMES sub-command. + Args: + command_parser (ArgumentParser): The command line sub-parser responsible for the HERMES sub-command. + + Returns: + None: """ pass - def load_settings(self, args: argparse.Namespace): - """Load settings from the configuration file (passed in from command line).""" + def load_settings(self: Self, args: argparse.Namespace) -> None: + """ + Load settings from the configuration file (passed in from command line). + + Args: + args (Namespace): The namespace that was returned by the command line parser when reading the arguments. + + Returns: + None: + """ toml_data = toml.load(args.path / args.config) self.root_settings = HermesCommand.settings_class.model_validate(toml_data) self.settings = getattr(self.root_settings, self.command_name) - def patch_settings(self, args: argparse.Namespace): - """Process command line options for the settings.""" + def patch_settings(self: Self, args: argparse.Namespace) -> None: + """ + Process command line options for the settings. + + Args: + args (Namespace): The namespace that was returned by the command line parser when reading the arguments. + + Returns: + None: + """ for key, value in args.options: target = self.settings @@ -148,27 +204,42 @@ def patch_settings(self, args: argparse.Namespace): setattr(target, sub_keys[-1], value) @abc.abstractmethod - def __call__(self, args: argparse.Namespace): + def __call__(self: Self, args: argparse.Namespace) -> None: """Execute the HERMES sub-command. - :param args: The namespace that was returned by the command line parser when reading the arguments. + Args: + args (Namespace): The namespace that was returned by the command line parser when reading the arguments. + + Returns: + None: """ pass class HermesPlugin(abc.ABC): - """Base class for all HERMES plugins.""" + """ + Base class for all HERMES plugins. + + Attributes: + plugin_node: ... + settings_class: The settings_class of the plugin. + """ pluing_node = None - settings_class: Union[Type, None] = None + settings_class: Optional[type] = None @abc.abstractmethod - def __call__(self, command: HermesCommand) -> None: - """Execute the plugin. + def __call__(self: Self, command: HermesCommand) -> None: + """ + Execute the plugin. + + Args: + command (HermesCommand): The command that triggered this plugin to run. - :param command: The command that triggered this plugin to run. + Returns: + None: """ pass @@ -180,12 +251,27 @@ class HermesHelpSettings(BaseModel): class HermesHelpCommand(HermesCommand): - """Show help page and exit.""" + """ + Show help page and exit. + + Attributes: + command_name (str): (class attribute) The name of the command. + settings_class (type): (class attribute) The settings class for general help settings. + """ command_name = "help" settings_class = HermesHelpSettings - def init_command_parser(self, command_parser: argparse.ArgumentParser) -> None: + def init_command_parser(self: Self, command_parser: argparse.ArgumentParser) -> None: + """ + Add arguments for help command. + + Args: + command_parser (ArgumentParser): The used argument parser. + + Returns: + None: + """ command_parser.add_argument( "subcommand", nargs="?", @@ -193,7 +279,16 @@ def init_command_parser(self, command_parser: argparse.ArgumentParser) -> None: help="The HERMES sub-command to get help for.", ) - def __call__(self, args: argparse.Namespace) -> None: + def __call__(self: Self, args: argparse.Namespace) -> None: + """ + Execute the hermes command `self`. + + Args: + args (Namespace): The namespace that was returned by the command line parser when reading the arguments. + + Returns: + None: + """ if args.subcommand: # When a sub-command is given, show its help page (i.e., by "running" the command with "-h" flag). self.parser.parse_args([args.subcommand, "-h"]) @@ -209,15 +304,38 @@ class HermesVersionSettings(BaseModel): class HermesVersionCommand(HermesCommand): - """Show HERMES version and exit.""" + """ + Show HERMES version and exit. + + Attributes: + command_name (str): (class attribute) The name of the command. + settings_class (type): (class attribute) The settings class for general help settings. + """ command_name = "version" settings_class = HermesVersionSettings - def load_settings(self, args: argparse.Namespace): - """Pass loading settings as not necessary for this command.""" + def load_settings(self: Self, args: argparse.Namespace) -> None: + """ + Pass loading settings as not necessary for this command. + + Args: + args (Namespace): The namespace that was returned by the command line parser when reading the arguments. + + Returns: + None: + """ pass - def __call__(self, args: argparse.Namespace) -> None: + def __call__(self: Self, args: argparse.Namespace) -> None: + """ + Execute the hermes command `self`. + + Args: + args (Namespace): The namespace that was returned by the command line parser when reading the arguments. + + Returns: + None: + """ self.log.info(metadata.version("hermes")) self.parser.exit() diff --git a/src/hermes/commands/curate/base.py b/src/hermes/commands/curate/base.py index e7de9a60..d87966da 100644 --- a/src/hermes/commands/curate/base.py +++ b/src/hermes/commands/curate/base.py @@ -52,7 +52,7 @@ class HermesCurateCommand(HermesCommand): Curate the unified metadata before deposition. Attributes: - args (Namespace): The arguments of the command. + args (Namespace): The namespace that was returned by the command line parser when reading the arguments. command_name (str): (class attribute) The name of the command. settings_class (type): (class attribute) The settings class for general curate settings. """ @@ -65,7 +65,7 @@ def __call__(self: Self, args: argparse.Namespace) -> None: Execute the hermes command `self`. Args: - args (Namespace): The arguments of the command. + args (Namespace): The namespace that was returned by the command line parser when reading the arguments. Returns: None: diff --git a/src/hermes/commands/deposit/base.py b/src/hermes/commands/deposit/base.py index ab5e5b62..e0f67aab 100644 --- a/src/hermes/commands/deposit/base.py +++ b/src/hermes/commands/deposit/base.py @@ -312,7 +312,7 @@ class HermesDepositCommand(HermesCommand): Deposit the curated metadata to repositories. Attributes: - args (Namespace): The arguments of the command. + args (Namespace): The namespace that was returned by the command line parser when reading the arguments. command_name (str): (class attribute) The name of the command. settings_class (type): (class attribute) The settings class for general deposit settings. """ @@ -342,7 +342,7 @@ def __call__(self: Self, args: argparse.Namespace) -> None: Execute the hermes command `self`. Args: - args (Namespace): The arguments of the command. + args (Namespace): The namespace that was returned by the command line parser when reading the arguments. Returns: None: diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index f1691e46..968a9c09 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -154,7 +154,7 @@ class HermesHarvestCommand(HermesCommand): Harvest metadata from configured sources. Attributes: - args (Namespace): The arguments of the command. + args (Namespace): The namespace that was returned by the command line parser when reading the arguments. command_name (str): (class attribute) The name of the command settings_class (type): (class attribute) The settings class for general harvest settings. """ @@ -167,7 +167,7 @@ def __call__(self: Self, args: argparse.Namespace) -> None: Execute the hermes command `self`. Args: - args (Namespace): The arguments of the command. + args (Namespace): The namespace that was returned by the command line parser when reading the arguments. Returns: None: diff --git a/src/hermes/commands/postprocess/base.py b/src/hermes/commands/postprocess/base.py index d960f8b9..9d1bf510 100644 --- a/src/hermes/commands/postprocess/base.py +++ b/src/hermes/commands/postprocess/base.py @@ -192,7 +192,7 @@ class HermesPostprocessCommand(HermesCommand): Post-process the published metadata after deposition. Attributes: - args (Namespace): The arguments of the command. + args (Namespace): The namespace that was returned by the command line parser when reading the arguments. command_name (str): (class attribute) The name of the command. settings_class (type): (class attribute) The settings class for general deposit settings. """ @@ -205,7 +205,7 @@ def __call__(self: Self, args: argparse.Namespace) -> None: Execute the hermes command `self`. Args: - args (Namespace): The arguments of the command. + args (Namespace): The namespace that was returned by the command line parser when reading the arguments. Returns: None: diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index c474fe65..357739fc 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -56,7 +56,7 @@ class HermesProcessCommand(HermesCommand): Process the collected metadata into a common dataset. Attributes: - args (Namespace): The arguments of the command. + args (Namespace): The namespace that was returned by the command line parser when reading the arguments. command_name (str): (class attribute) The name of the command settings_class (type): (class attribute) The settings class for general process settings. """ @@ -69,7 +69,7 @@ def __call__(self: Self, args: argparse.Namespace) -> None: Execute the hermes command `self`. Args: - args (Namespace): The arguments of the command. + args (Namespace): The namespace that was returned by the command line parser when reading the arguments. Returns: None: diff --git a/src/hermes/commands/report/base.py b/src/hermes/commands/report/base.py index abc0ad15..653c085d 100644 --- a/src/hermes/commands/report/base.py +++ b/src/hermes/commands/report/base.py @@ -15,17 +15,32 @@ class HermesReportSettings(BaseModel): - """Configuration of the ``report`` command.""" + """ Configuration of the ``report`` command. """ pass class HermesReportCommand(HermesCommand): - """ Gernate a summarized provenance report for the steps chosen by the user. """ + """ + Generate a summarized provenance report for the steps chosen by the user. - command_name = "report" - settings_class = HermesReportSettings + Attributes: + command_name (str): (class attribute) The name of the command. + settings_class (type): (class attribute) The settings class for general report settings. + """ + + command_name: str = "report" + settings_class: type = HermesReportSettings def init_command_parser(self: Self, command_parser: argparse.ArgumentParser) -> None: + """ + Add arguments for report command. + + Args: + command_parser (ArgumentParser): The used argument parser. + + Returns: + None: + """ command_parser.add_argument( "--steps", nargs="*", @@ -35,9 +50,21 @@ def init_command_parser(self: Self, command_parser: argparse.ArgumentParser) -> ) def __call__(self: Self, args: argparse.Namespace) -> None: + """ + Execute the hermes command `self`. + + Args: + args (Namespace): The namespace that was returned by the command line parser when reading the arguments. + + Returns: + None: + """ print("\nProvenance report for HERMES:") + # print the report for every step for step in args.steps: + # reset ld_prov_list because it is usually a singelton ld_prov_list.INDICES = {} + # print the report match step: case "harvest": self.report_harvest() @@ -52,7 +79,14 @@ def __call__(self: Self, args: argparse.Namespace) -> None: print("") def report_harvest(self: Self) -> None: + """ + Print the report for the harvest step. + + Returns: + None: + """ print("- Harvest:") + # load provenance data or error out ctx = HermesContext() ctx.prepare_step("harvest") with ctx["provenance"] as cache: @@ -63,17 +97,21 @@ def report_harvest(self: Self) -> None: return finally: ctx.finalize_step("harvest") + # get basic hermes objects harvest_base_plugin = prov_doc.get_hermes_base_plugin("harvest") harvest_command = prov_doc.get_hermes_command("harvest") hermes_cache = prov_doc.get_hermes_cache() plugins = prov_doc.shallow_search(lambda node: ( "prov:actedOnBehalfOf" in node and node["prov:actedOnBehalfOf"] == [harvest_base_plugin.ref] )) + # for every plugin print the info on this plugins execution for plugin in plugins: + # print basic info on the plugin print( f" - Plugin {plugin['@id'][24:]} ({plugin['schema:name'][0]}, version " f"{vers if (vers := plugin.get('schema:softwareVersion', False)) else 'N/A'})" ) + # print every source loaded by the plugin print(" - Loaded data from:") for load_action in prov_doc.shallow_search(lambda node: ( "prov:wasAssociatedWith" in node and @@ -85,6 +123,7 @@ def report_harvest(self: Self) -> None: f" - {source['schema:url'][0]} (at {load_action['prov:startedAtTime'][0]}, took " + f"{load_action['prov:endedAtTime'][0]-load_action['prov:startedAtTime'][0]})" ) + # print infos on the result store_action = prov_doc.shallow_search(lambda node: ( "prov:wasAssociatedWith" in node and node["prov:wasAssociatedWith"] == [plugin.ref, hermes_cache.ref, harvest_command.ref] @@ -99,7 +138,14 @@ def report_harvest(self: Self) -> None: print(f" - {result['schema:url'][0]} ({result['schema:description'][0].split(' ')[1]})") def report_process(self: Self) -> None: + """ + Print the report for the process step. + + Returns: + None: + """ print("- Process:") + # load provenance data or error out ctx = HermesContext() ctx.prepare_step("process") with ctx["provenance"] as cache: @@ -110,10 +156,14 @@ def report_process(self: Self) -> None: return finally: ctx.finalize_step("process") + # get basic hermes objects process_base_plugin = prov_doc.get_hermes_base_plugin("process") + process_command = prov_doc.get_hermes_command("process") + hermes_cache = prov_doc.get_hermes_cache() plugins = prov_doc.shallow_search(lambda node: ( "prov:actedOnBehalfOf" in node and node["prov:actedOnBehalfOf"] == [process_base_plugin.ref] )) + # print info on all plugins and their strategy generation for plugin in plugins: print( f" - Plugin {plugin['@id'][24:]} ({plugin['schema:name'][0]}, version " @@ -126,14 +176,13 @@ def report_process(self: Self) -> None: f" - Generated strategies at {strategy_generation['prov:startedAtTime'][0]} took " f"{strategy_generation['prov:endedAtTime'][0]-strategy_generation['prov:startedAtTime'][0]}" ) - process_command = prov_doc.get_hermes_command("process") - hermes_cache = prov_doc.get_hermes_cache() load_actions = prov_doc.shallow_search(lambda node: ( "prov:wasAssociatedWith" in node and node["prov:wasAssociatedWith"] == [hermes_cache.ref, process_command.ref] and "prov:used" in node and len(node["prov:used"]) == 3 )) + # print info on the data loaded that was merged for index, load_action in enumerate(sorted(load_actions, key=lambda it: it["prov:startedAtTime"][0]), start=1): print( f" - In load {index} loaded (at {load_action['prov:startedAtTime'][0]}, took" @@ -150,6 +199,7 @@ def report_process(self: Self) -> None: "prov:wasInformedBy" in node and len(node["prov:wasInformedBy"]) == 3 )) + # print info on the merges for index, merger in enumerate(sorted(bigest_mergers, key=lambda it: it["prov:startedAtTime"][0]), start=1): if index == 1: merged = "merged data from load 1 with data of load 2" @@ -168,6 +218,7 @@ def report_process(self: Self) -> None: stored_objects = prov_doc.shallow_search(lambda node: ( "prov:wasGeneratedBy" in node and node["prov:wasGeneratedBy"] == [write_action.ref] )) + # print info on the stored data print( f" - Results stored (at {write_action['prov:startedAtTime'][0]} took " f"{write_action['prov:endedAtTime'][0]-write_action['prov:startedAtTime'][0]}) in:" @@ -176,7 +227,14 @@ def report_process(self: Self) -> None: print(f" - {res['schema:url'][0]} ({res['schema:description'][0].split(' ')[1]})") def report_curate(self: Self) -> None: + """ + Print the report for the curate step. + + Returns: + None: + """ print("- Curate:") + # load provenance data or error out ctx = HermesContext() ctx.prepare_step("curate") with ctx["provenance"] as cache: @@ -187,7 +245,11 @@ def report_curate(self: Self) -> None: return finally: ctx.finalize_step("curate") + # get basic hermes objects curate_base_plugin = prov_doc.get_hermes_base_plugin("curate") + process_command = prov_doc.get_hermes_command("process") + hermes_cache = prov_doc.get_hermes_cache() + # print info on the used plugin curate_plugin = prov_doc.shallow_search(lambda node: ( "prov:actedOnBehalfOf" in node and node["prov:actedOnBehalfOf"] == [curate_base_plugin.ref] ))[0] @@ -195,8 +257,7 @@ def report_curate(self: Self) -> None: f" - Plugin used:\n - {curate_plugin['@id'][23:]} ({curate_plugin['schema:name'][0]}, version " f"{vers if (vers := curate_plugin.get('schema:softwareVersion', False)) else 'N/A'})" ) - process_command = prov_doc.get_hermes_command("process") - hermes_cache = prov_doc.get_hermes_cache() + # get objects that contain info on the curation store_action_of_process = prov_doc.shallow_search(lambda node: ( "prov:wasAssociatedWith" in node and node["prov:wasAssociatedWith"] == [process_command.ref, hermes_cache.ref] and @@ -217,6 +278,7 @@ def report_curate(self: Self) -> None: write = prov_doc.shallow_search(lambda node: ( "prov:used" in node and node["prov:used"] == [curate_activity.ref] ))[0] + # print curation info print( f" - Time consumed:\n - Curation at ~{load_action['prov:endedAtTime'][0]} took" f" ~{curate_activity['prov:generatedAtTime'][0]-load_action['prov:endedAtTime'][0]}" @@ -236,7 +298,14 @@ def report_curate(self: Self) -> None: print(f" - {result['schema:url'][0]} ({result['schema:description'][0].split(' ')[1]})") def report_deposit(self: Self) -> None: + """ + Print the report for the deposit step. + + Returns: + None: + """ print("- Deposit:") + # load provenance data or error out ctx = HermesContext() ctx.prepare_step("deposit") with ctx["provenance"] as cache: @@ -247,7 +316,11 @@ def report_deposit(self: Self) -> None: return finally: ctx.finalize_step("deposit") + # get basic hermes objects deposit_base_plugin = prov_doc.get_hermes_base_plugin("deposit") + curate_command = prov_doc.get_hermes_command("curate") + hermes_cache = prov_doc.get_hermes_cache() + # print info on the plugin deposit_plugin = prov_doc.shallow_search(lambda node: ( "prov:actedOnBehalfOf" in node and node["prov:actedOnBehalfOf"] == [deposit_base_plugin.ref] ))[0] @@ -255,8 +328,7 @@ def report_deposit(self: Self) -> None: f" - Plugin used:\n - {deposit_plugin['@id'][24:]} ({deposit_plugin['schema:name'][0]}, version " f"{vers if (vers := deposit_plugin.get('schema:softwareVersion', False)) else 'N/A'})" ) - curate_command = prov_doc.get_hermes_command("curate") - hermes_cache = prov_doc.get_hermes_cache() + # get objects containing info on map and update of the metadata store_action_of_curate = prov_doc.shallow_search(lambda node: ( "prov:wasAssociatedWith" in node and node["prov:wasAssociatedWith"] == [curate_command.ref, hermes_cache.ref] and @@ -290,6 +362,7 @@ def report_deposit(self: Self) -> None: map_action = prov_doc.shallow_search(lambda node: ( "@id" in node and node["@id"] == mapped_metadata["prov:wasGeneratedBy"][0]["@id"] ))[0] + # print general info and info on map as well as update of the metadata print( " - Time consumed:\n" f" - Preparation at ~{load_action['prov:endedAtTime'][0]} took ~" @@ -306,6 +379,7 @@ def report_deposit(self: Self) -> None: ) for source in stored_results_of_curate: print(4*" " + f"- {source['schema:url'][0]} ({source['schema:description'][0].split(' ')[1]})") + # print info on the store of the mapped as well as updated metadata print( f" - Metadata mapped for deposit stored (at {store_mapped['prov:startedAtTime'][0]}, took " f"{store_mapped['prov:endedAtTime'][0]-store_mapped['prov:startedAtTime'][0]}) in:\n" @@ -316,7 +390,14 @@ def report_deposit(self: Self) -> None: ) def report_postprocess(self: Self) -> None: + """ + Print the report for the harvest step. + + Returns: + None: + """ print("- Postprocess:") + # load provenance data or error out ctx = HermesContext() ctx.prepare_step("postprocess") with ctx["provenance"] as cache: @@ -327,9 +408,11 @@ def report_postprocess(self: Self) -> None: return finally: ctx.finalize_step("postprocess") + # get basic hermes objects cache = prov_doc.get_hermes_cache() command = prov_doc.get_hermes_command("postprocess") base_plugin = prov_doc.get_hermes_base_plugin("postprocess") + # print info on the plugin plugin = prov_doc.shallow_search(lambda node: ( "prov:actedOnBehalfOf" in node and node["prov:actedOnBehalfOf"] == [base_plugin.ref] ))[0] @@ -341,6 +424,7 @@ def report_postprocess(self: Self) -> None: "prov:wasAssociatedWith" in node and node["prov:wasAssociatedWith"] == [plugin.ref, base_plugin.ref, command.ref, cache.ref] )) + # print info on the loads from cache print(" - Used deposit results:") for index, cache_load in enumerate(cache_loads, start=1): source_id = cache_load["prov:used"][0]["@id"] @@ -354,6 +438,7 @@ def report_postprocess(self: Self) -> None: "prov:wasAssociatedWith" in node and node["prov:wasAssociatedWith"] == [plugin.ref, base_plugin.ref, command.ref] )) + # sort io operations into load and write operations loads, writes = [], [] for io_op in io_ops: used = io_op["prov:used"][0]["@id"] @@ -361,6 +446,7 @@ def report_postprocess(self: Self) -> None: writes.append(io_op) else: loads.append(io_op) + # print info on general loads of the plugin print(" - Loaded data from:") for index, load in enumerate(loads): source_id = load["prov:used"][0]["@id"] @@ -370,6 +456,7 @@ def report_postprocess(self: Self) -> None: f"{load['prov:endedAtTime']-load['prov:startedAtTime']} from:\n" f" - {source['schema:url']}" ) + # print info on general writes of the plugin print(" - Written data to:") for index, write in enumerate(writes): target = prov_doc.shallow_search(lambda node: ( From b4091201806486f4912c4f2708827b21462a31bf Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Mon, 31 Aug 2026 15:12:36 +0200 Subject: [PATCH 30/41] fixed small issues and typos --- docs/adr/0012-overall-data-model-design.md | 4 +-- src/hermes/commands/process/base.py | 2 +- src/hermes/commands/process/invenio_merge.py | 18 +++++----- src/hermes/model/error.py | 8 ++--- src/hermes/model/merge/action.py | 4 ++- src/hermes/model/merge/container.py | 3 -- src/hermes/model/types/ld_container.py | 3 +- src/hermes/model/types/ld_context.py | 2 +- src/hermes/model/types/ld_dict.py | 34 ++++++++++++++++++ src/hermes/model/types/ld_list.py | 5 ++- test/hermes_test/model/test_api.py | 1 - .../model/types/test_ld_container.py | 8 +++-- test/hermes_test/model/types/test_ld_dict.py | 35 +++++++++++++++++++ 13 files changed, 99 insertions(+), 28 deletions(-) diff --git a/docs/adr/0012-overall-data-model-design.md b/docs/adr/0012-overall-data-model-design.md index 2347f532..5aae2867 100644 --- a/docs/adr/0012-overall-data-model-design.md +++ b/docs/adr/0012-overall-data-model-design.md @@ -22,13 +22,13 @@ Superseded: we no longer need to serialize additional information like provenanc ## Considered Options * One common model for all stages -* Seperate model for different stages +* Separate model for different stages * Common model for all stages * Processing model and curated model ## Decision Outcome -Chosen option: "Seperate model for different stages", because comes out best. +Chosen option: "Separate model for different stages", because comes out best. ## Pros and Cons of the Options diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index 357739fc..0263ab11 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -237,7 +237,7 @@ def add_strategies_to_merge_doc( "prov:startedAtTime": generate_strategies_start, "prov:endedAtTime": generate_strategies_end }) - new_strategies = prov_doc.add_entity(data={ # TODO: record strategies + new_strategies = prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": f"new merge strategies of plugin {plugin_name}", "schema:text": str(additional_strategies), # TODO: maybe "prov:value" instead? diff --git a/src/hermes/commands/process/invenio_merge.py b/src/hermes/commands/process/invenio_merge.py index f5e27034..d36ee2da 100644 --- a/src/hermes/commands/process/invenio_merge.py +++ b/src/hermes/commands/process/invenio_merge.py @@ -36,15 +36,15 @@ def merge( isinstance(value[0], (dict, ld_merge_dict)) and [*value[0].keys()] == ["@id"] ): if value != update: - target.reject(key, update) + target.reject(key[-1], update) return value if isinstance(update, ld_list) and len(update) == 1: if isinstance(update[0], str) or ( isinstance(update[0], (dict, ld_merge_dict)) and [*update[0].keys()] == ["@id"] ): - target.replace(key, value) + target.replace(key[-1], value) return update - target.reject(key, update) + target.reject(key[-1], update) return value if ( (key[-1] == iri["schema:familyName"] and iri["schema:Person"] in types) or @@ -54,14 +54,14 @@ def merge( ): if len(value) == 1: if value != update: - target.reject(key, update) + target.reject(key[-1], update) return value if len(update) == 1: - target.replace(key, value) + target.replace(key[-1], value) return update if len(value) == len(update) == 0: return value - target.reject(key, update) + target.reject(key[-1], update) return value if ( (key[-1] == iri["schema:version"] or key[-1] == iri["schema:description"]) and @@ -69,14 +69,14 @@ def merge( ): if len(value) == 1: if value != update: - target.reject(key, update) + target.reject(key[-1], update) return value if len(update) == 1: - target.replace(key, value) + target.replace(key[-1], value) return update if len(value) == 0 or len(update) == 0: return [] - target.reject(key, update) + target.reject(key[-1], update) return value diff --git a/src/hermes/model/error.py b/src/hermes/model/error.py index 1318420d..64dc6f0c 100644 --- a/src/hermes/model/error.py +++ b/src/hermes/model/error.py @@ -47,9 +47,9 @@ class HermesMergeError(Exception): This exception should be raised when there is an error during a merge / set operation. Attributes: - path (list[str | int]): The path where the merge error occured. + path (list[str | int]): The path where the merge error occurred. old_value (Any): Old value that was stored at `path`. - new_value (Any): New value that was to be assinged. + new_value (Any): New value that was to be assigned . tag: Tag data for the new value. """ def __init__(self, path: list[Union[str, int]], old_value: Any, new_value: Any, **kwargs) -> None: @@ -57,9 +57,9 @@ def __init__(self, path: list[Union[str, int]], old_value: Any, new_value: Any, Create a new merge incident. Args: - path (list[str | int]): The path where the merge error occured. + path (list[str | int]): The path where the merge error occurred. old_value (Any): Old value that was stored at `path`. - new_value (Any): New value that was to be assinged. + new_value (Any): New value that was to be assigned . kwargs: Tag data for the new value. Returns: diff --git a/src/hermes/model/merge/action.py b/src/hermes/model/merge/action.py index 9c52115c..c95b7ef3 100644 --- a/src/hermes/model/merge/action.py +++ b/src/hermes/model/merge/action.py @@ -282,7 +282,9 @@ def merge( elif isinstance(item, ld_list) and isinstance(update_item, ld_list): self.merge(target, [*key, index], item, update_item) elif isinstance(item, (ld_dict, ld_list)) or isinstance(update_item, (ld_dict, ld_list)): - """ FIXME: log error """ + """ + FIXME: log error/ warning that merge of items at... could not be merged and will be skipped + """ break else: value.append(update_item) diff --git a/src/hermes/model/merge/container.py b/src/hermes/model/merge/container.py index 64cb7ac0..000c8c02 100644 --- a/src/hermes/model/merge/container.py +++ b/src/hermes/model/merge/container.py @@ -373,7 +373,6 @@ def _add_related( Returns: None: """ - # FIXME: key not only string # make sure appending is possible self.emplace(rel) # append the new entry @@ -392,7 +391,6 @@ def reject(self: Self, key: str, value: Union[BASIC_TYPE, TIME_TYPE, ld_dict, ld Returns: None: """ - # FIXME: key not only string self._add_related("hermes-rt:reject", key, value) def replace(self: Self, key: str, value: Union[BASIC_TYPE, TIME_TYPE, ld_dict, ld_list]) -> None: @@ -408,5 +406,4 @@ def replace(self: Self, key: str, value: Union[BASIC_TYPE, TIME_TYPE, ld_dict, l Returns: None: """ - # FIXME: key not only string self._add_related("hermes-rt:replace", key, value) diff --git a/src/hermes/model/types/ld_container.py b/src/hermes/model/types/ld_container.py index 19fd70ae..cc4a346f 100644 --- a/src/hermes/model/types/ld_container.py +++ b/src/hermes/model/types/ld_container.py @@ -232,7 +232,7 @@ def _to_expanded_json( # all ld_container (ld_dicts and ld_lists) and datetime, date as well as time objects in value have to dissolved # because the JSON-LD processor can't handle them # to do this traverse value in a BFS and replace all items with a type in 'special_types' with a usable values - key_and_reference_todo_list = [(0, [value])] + key_and_reference_todo_list: list[Union[tuple[int, list], tuple[str, dict]]] = [(0, [value])] special_types = (list, dict, ld_container, datetime, date, time) while True: # check if ready @@ -462,7 +462,6 @@ def typed_ld_to_py(cls: type[Self], data: list[dict[str, BASIC_TYPE]], **kwargs) Returns: BASIC_TYPE | TIME_TYPE: The pythonized version of data. """ - # FIXME: #434 dates are not returned as datetime/ date/ time but as string ld_value = data[0]['@value'] if iri_map["schema:DateTime"] == data[0]['@type']: ld_value = datetime.fromisoformat(ld_value) diff --git a/src/hermes/model/types/ld_context.py b/src/hermes/model/types/ld_context.py index 681f4792..8e7d40bd 100644 --- a/src/hermes/model/types/ld_context.py +++ b/src/hermes/model/types/ld_context.py @@ -128,7 +128,7 @@ def __getitem__(self: Self, compressed_term: Union[str, tuple]) -> str: Raises: HermesContextError: If the compressed term is '' or its prefix can't be expanded. """ - # seperate the prefix from the term + # separate the prefix from the term if not isinstance(compressed_term, str): prefix, term = compressed_term elif ":" in compressed_term: diff --git a/src/hermes/model/types/ld_dict.py b/src/hermes/model/types/ld_dict.py index 4d419ad7..0514ee21 100644 --- a/src/hermes/model/types/ld_dict.py +++ b/src/hermes/model/types/ld_dict.py @@ -8,6 +8,7 @@ from __future__ import annotations from collections.abc import Generator, Iterator, KeysView +from datetime import date, datetime, time from typing import Any, Literal, Optional, Union, TYPE_CHECKING from typing_extensions import Self @@ -383,6 +384,39 @@ def from_dict( Returns: ld_dict: The new ld_dict build from value. """ + # all ld_container (ld_dicts and ld_lists) and datetime, date as well as time objects in value have to dissolved + # because the JSON-LD processor can't handle them + # to do this traverse value in a BFS and replace all items with a type in 'special_types' with a usable values + key_and_reference_todo_list: list[Union[tuple[int, list], tuple[str, dict]]] = [(0, [value])] + special_types = (list, dict, ld_container, datetime, date, time) + while True: + # check if ready + if len(key_and_reference_todo_list) == 0: + break + # get next item + tmp_key, ref = key_and_reference_todo_list.pop() + temp = ref[tmp_key] + # replace item if necessary and add childs to the todo list + if isinstance(temp, list): + key_and_reference_todo_list.extend( + [(index, temp) for index, val in enumerate(temp) if isinstance(val, special_types)] + ) + elif isinstance(temp, dict): + key_and_reference_todo_list.extend( + [(new_key, temp) for new_key in temp.keys() if isinstance(temp[new_key], special_types)] + ) + elif isinstance(temp, ld_container): + if "ld_list" in [sub_cls.__name__ for sub_cls in type(temp).mro()] and temp.container_type == "@set": + ref[tmp_key] = temp._data + else: + ref[tmp_key] = temp._data[0] + elif isinstance(temp, datetime): + ref[tmp_key] = {"@value": temp.isoformat(), "@type": "schema:DateTime"} + elif isinstance(temp, date): + ref[tmp_key] = {"@value": temp.isoformat(), "@type": "schema:Date"} + elif isinstance(temp, time): + ref[tmp_key] = {"@value": temp.isoformat(), "@type": "schema:Time"} + # make a copy of value and add the new type to it. ld_data = value.copy() ld_type = ld_container.merge_to_list(ld_type or [], ld_data.get('@type', [])) diff --git a/src/hermes/model/types/ld_list.py b/src/hermes/model/types/ld_list.py index 01a1c265..7cbe8c81 100644 --- a/src/hermes/model/types/ld_list.py +++ b/src/hermes/model/types/ld_list.py @@ -655,7 +655,6 @@ def from_list( Raises: ValueError: If key is '@type' and container_type is not '@set'. """ - # TODO: handle context if not of type list or None # validate container_type if key == "@type": if container_type != "@set": @@ -667,6 +666,10 @@ def from_list( elif container_type != "@set": raise ValueError(f"Invalid container type: {container_type}. (valid are only '@set', '@list' and '@graph')") + # handle non-list context + if context is not None and not isinstance(context, list): + context = [context] + if parent is not None: # expand value in the "context" of parent if isinstance(parent, ld_list): diff --git a/test/hermes_test/model/test_api.py b/test/hermes_test/model/test_api.py index 906203b5..0b97b6b4 100644 --- a/test/hermes_test/model/test_api.py +++ b/test/hermes_test/model/test_api.py @@ -143,6 +143,5 @@ def test_usage(): if "Baz" not in author["name"]: assert "email" in author if "schema:knowsAbout" not in author: - # FIXME: None has to be discussed author["schema:knowsAbout"] = None author["schema:pronouns"] = "they/them" diff --git a/test/hermes_test/model/types/test_ld_container.py b/test/hermes_test/model/types/test_ld_container.py index f0844ecd..dbab79e8 100644 --- a/test/hermes_test/model/types/test_ld_container.py +++ b/test/hermes_test/model/types/test_ld_container.py @@ -117,9 +117,11 @@ def test_to_python_basic_value(self, mock_context): def test_to_python_datetime_value(self, mock_context): cont = ld_container([{}], context=[mock_context]) - assert cont._to_python("http://spam.eggs/eggs", { - "@value": "2022-02-22T00:00:00", "@type": "https://schema.org/DateTime" - }) == "2022-02-22T00:00:00" # TODO: #434 typed date is returned as string instead of date + res = cont._to_python("http://spam.eggs/eggs", { + "@value": "2022-02-22T00:00:00", "@type": "http://schema.org/DateTime" + }) + assert isinstance(res, datetime) + assert res == datetime.fromisoformat("2022-02-22T00:00:00") def test_to_python_error(self, mock_context): cont = ld_container([{}], context=[mock_context]) diff --git a/test/hermes_test/model/types/test_ld_dict.py b/test/hermes_test/model/types/test_ld_dict.py index 66ce44cb..05fd2762 100644 --- a/test/hermes_test/model/types/test_ld_dict.py +++ b/test/hermes_test/model/types/test_ld_dict.py @@ -5,6 +5,8 @@ # SPDX-FileContributor: Stephan Druskat # SPDX-FileContributor: Michael Fritzsche +from datetime import datetime + import pytest from hermes.model.types.ld_dict import ld_dict @@ -373,6 +375,39 @@ def test_from_dict(): assert di["http://xmlns.com/foaf/0.1/name"] == di["xmlns:name"] == ["fo"] assert di.context == [{"schema": "https://schema.org/"}, {"xmlns": "http://xmlns.com/foaf/0.1/"}] + di = ld_dict.from_dict( + { + "@context": {"schema": "http://schema.org/"}, + "@type": "schema:Thing", + "schema:owner": ld_dict.from_dict( + { + "@context": {"schema": "http://schema.org/"}, + "@type": "schema:Person", + "schema:name": "Foo" + } + ) + } + ) + assert di["schema:owner"][0]["schema:name"][0] == "Foo" + di = ld_dict.from_dict( + { + "@context": {"schema": "http://schema.org/"}, + "@type": "schema:Thing", + "schema:name": ld_list.from_list( + ["Foo", "Bar"], key="schema:name", context={"schema": "http://schema.org/"}, container_type="@list" + ) + } + ) + assert di["schema:name"][0] == "Foo" + di = ld_dict.from_dict( + { + "@context": {"schema": "http://schema.org/"}, + "@type": "schema:CreativeWork", + "schema:dateCreated": datetime(2026, 8, 31, 14, 50) + } + ) + assert di["schema:dateCreated"][0] == datetime(2026, 8, 31, 14, 50) + def test_is_ld_dict(): assert not any(ld_dict.is_ld_dict(item) for item in [{}, {"foo": "bar"}, {"@id": "foo"}]) From f3eb1677e833ad939bd77cb296be36ec79db58a8 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Mon, 31 Aug 2026 15:19:14 +0200 Subject: [PATCH 31/41] remove some outdated fixme comments --- src/hermes/commands/curate/base.py | 10 +++++----- src/hermes/commands/deposit/base.py | 10 +++++----- src/hermes/commands/harvest/base.py | 8 ++++---- src/hermes/commands/process/base.py | 12 ++++++------ src/hermes/model/merge/container.py | 2 +- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/hermes/commands/curate/base.py b/src/hermes/commands/curate/base.py index d87966da..995cc3d4 100644 --- a/src/hermes/commands/curate/base.py +++ b/src/hermes/commands/curate/base.py @@ -159,7 +159,7 @@ def __call__(self: Self, args: argparse.Namespace) -> None: loaded_data = prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "data loaded from process step", - "schema:text": loaded_metadata_str, # TODO: maybe "prov:value" instead? + "schema:text": loaded_metadata_str, "prov:wasAttributedTo": hermes_cache.ref, "prov:wasGeneratedBy": load_action.ref, "prov:wasDerivedFrom": stored_results_of_process, @@ -168,7 +168,7 @@ def __call__(self: Self, args: argparse.Namespace) -> None: curated_data = prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "curated metadata", - "schema:text": str(curated_metadata.compact()), # TODO: maybe "prov:value" instead? + "schema:text": str(curated_metadata.compact()), "prov:wasAttributedTo": [curate_plugin.ref, curate_base_plugin.ref, curate_command.ref], "prov:wasInfluencedBy": curate_plugin.ref, "prov:wasGeneratedBy": load_action.ref, @@ -186,7 +186,7 @@ def __call__(self: Self, args: argparse.Namespace) -> None: prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "The compacted version of the processed metadata.", - "schema:text": str(curated_metadata.compact()), # TODO: maybe "prov:value" instead? + "schema:text": str(curated_metadata.compact()), "schema:encodingFormat": "application/json", "schema:url": (ctx.cache_dir / "curate" / "result" / "codemeta.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, @@ -197,7 +197,7 @@ def __call__(self: Self, args: argparse.Namespace) -> None: prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "The context of the processed metadata.", - "schema:text": str({"@context": curated_metadata.full_context}), # TODO: maybe "prov:value" instead? + "schema:text": str({"@context": curated_metadata.full_context}), "schema:encodingFormat": "application/json", "schema:url": (ctx.cache_dir / "curate" / "result" / "context.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, @@ -208,7 +208,7 @@ def __call__(self: Self, args: argparse.Namespace) -> None: prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "The expanded version of the processed metadata.", - "schema:text": str(curated_metadata.ld_value), # TODO: maybe "prov:value" instead? + "schema:text": str(curated_metadata.ld_value), "schema:encodingFormat": "application/json", "schema:url": (ctx.cache_dir / "curate" / "result" / "expanded.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, diff --git a/src/hermes/commands/deposit/base.py b/src/hermes/commands/deposit/base.py index e0f67aab..4dc6bb04 100644 --- a/src/hermes/commands/deposit/base.py +++ b/src/hermes/commands/deposit/base.py @@ -93,7 +93,7 @@ def __call__(self: Self, command: "HermesDepositCommand", prov_doc: Optional[ld_ loaded_data = prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "data loaded from curate step", - "schema:text": str(self.metadata.compact()), # TODO: maybe "prov:value" instead? + "schema:text": str(self.metadata.compact()), "prov:wasAttributedTo": hermes_cache.ref, "prov:wasGeneratedBy": load_action.ref, "prov:wasDerivedFrom": results_curate, @@ -122,7 +122,7 @@ def __call__(self: Self, command: "HermesDepositCommand", prov_doc: Optional[ld_ mapped_data = prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "The metadata mapped to the format required by the deposition target.", - "schema:text": str(deposit), # TODO: maybe "prov:value" instead? + "schema:text": str(deposit), "prov:wasAttributedTo": plugin.ref, "prov:wasGeneratedBy": map_action.ref, "prov:wasDerivedFrom": loaded_data.ref, @@ -138,7 +138,7 @@ def __call__(self: Self, command: "HermesDepositCommand", prov_doc: Optional[ld_ prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "The stored version of the mapped metadata.", - "schema:text": str(deposit), # TODO: maybe "prov:value" instead? + "schema:text": str(deposit), "schema:encodingFormat": "application/json", "schema:url": (ctx.cache_dir / "deposit" / target / "deposit.json").absolute().as_uri(), "prov:wasGeneratedBy": store_mapped_data.ref, @@ -166,7 +166,7 @@ def __call__(self: Self, command: "HermesDepositCommand", prov_doc: Optional[ld_ updated_mapped_data = prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "The updated mapped metadata.", - "schema:text": str(updated_deposit), # TODO: maybe "prov:value" instead? + "schema:text": str(updated_deposit), "prov:wasInfluencedBy": plugin.ref, "prov:wasDerivedFrom": mapped_data.ref, "prov:generatedAtTime": end_of_update_map @@ -181,7 +181,7 @@ def __call__(self: Self, command: "HermesDepositCommand", prov_doc: Optional[ld_ prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "The stored version of the updated mapped metadata.", - "schema:text": str(updated_deposit), # TODO: maybe "prov:value" instead? + "schema:text": str(updated_deposit), "schema:encodingFormat": "application/json", "schema:url": (ctx.cache_dir / "deposit" / target / "result.json").absolute().as_uri(), "prov:wasGeneratedBy": store_updated_mapped_data.ref, diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index 968a9c09..4f12d600 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -256,7 +256,7 @@ def __call__(self: Self, args: argparse.Namespace) -> None: data_output = prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "the harvested metadata", - "schema:text": str(harvested_data.compact()), # TODO: maybe "prov:value" instead? + "schema:text": str(harvested_data.compact()), "prov:wasAttributedTo": plugin.ref, "prov:wasGeneratedBy": map_activity.ref, "prov:wasDerivedFrom": outputs, @@ -279,7 +279,7 @@ def __call__(self: Self, args: argparse.Namespace) -> None: prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "The compacted version of the harvested metadata.", - "schema:text": str(harvested_data.compact()), # TODO: maybe "prov:value" instead? + "schema:text": str(harvested_data.compact()), "schema:encodingFormat": "application/json", "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "codemeta.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, @@ -290,7 +290,7 @@ def __call__(self: Self, args: argparse.Namespace) -> None: prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "The context of the harvested metadata.", - "schema:text": str({"@context": harvested_data.full_context}), # TODO: maybe "prov:value" instead? + "schema:text": str({"@context": harvested_data.full_context}), "schema:encodingFormat": "application/json", "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "context.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, @@ -301,7 +301,7 @@ def __call__(self: Self, args: argparse.Namespace) -> None: prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "The expanded version of the harvested metadata.", - "schema:text": str(harvested_data.ld_value), # TODO: maybe "prov:value" instead? + "schema:text": str(harvested_data.ld_value), "schema:encodingFormat": "application/json", "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "expanded.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index 0263ab11..c8c89a73 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -140,7 +140,7 @@ def __call__(self: Self, args: argparse.Namespace) -> None: prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "The compacted version of the processed metadata.", - "schema:text": str(merge_doc.compact()), # TODO: maybe "prov:value" instead? + "schema:text": str(merge_doc.compact()), "schema:encodingFormat": "application/json", "schema:url": (ctx.cache_dir / "process" / "result" / "codemeta.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, @@ -151,7 +151,7 @@ def __call__(self: Self, args: argparse.Namespace) -> None: prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "The context of the processed metadata.", - "schema:text": str({"@context": merge_doc.full_context}), # TODO: maybe "prov:value" instead? + "schema:text": str({"@context": merge_doc.full_context}), "schema:encodingFormat": "application/json", "schema:url": (ctx.cache_dir / "process" / "result" / "context.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, @@ -162,7 +162,7 @@ def __call__(self: Self, args: argparse.Namespace) -> None: prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "The expanded version of the processed metadata.", - "schema:text": str(merge_doc.ld_value), # TODO: maybe "prov:value" instead? + "schema:text": str(merge_doc.ld_value), "schema:encodingFormat": "application/json", "schema:url": (ctx.cache_dir / "process" / "result" / "expanded.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, @@ -240,7 +240,7 @@ def add_strategies_to_merge_doc( new_strategies = prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": f"new merge strategies of plugin {plugin_name}", - "schema:text": str(additional_strategies), # TODO: maybe "prov:value" instead? + "schema:text": str(additional_strategies), "prov:wasAttributedTo": plugin.ref, "prov:wasGeneratedBy": new_strategy_generation.ref, "prov:generatedAtTime": generate_strategies_end @@ -261,7 +261,7 @@ def add_strategies_to_merge_doc( }) merged_strategies = prov_doc.add_entity(data={ "schema:description": "the merge strategies of multiple plugins merged together", - "schema:text": str(merge_doc.strategies), # TODO: maybe "prov:value" instead? + "schema:text": str(merge_doc.strategies), "prov:wasDerivedFrom": [merged_strategies.ref, new_strategies.ref], "prov:wasGeneratedBy": strategy_action.ref, "prov:wasAttributedTo": process_command.ref, @@ -348,7 +348,7 @@ def merge_data_from_harvesters( new_data = prov_doc.add_entity(data={ # new data to be merged "@type": "schema:CreativeWork", "schema:description": f"data loaded from {harvester} plugin", - "schema:text": str(metadata.compact()), # TODO: maybe "prov:value" instead? + "schema:text": str(metadata.compact()), "prov:wasAttributedTo": [process_command.ref, hermes_cache.ref], "prov:wasGeneratedBy": new_action.ref, "prov:wasDerivedFrom": stored_results, diff --git a/src/hermes/model/merge/container.py b/src/hermes/model/merge/container.py index 000c8c02..b577757e 100644 --- a/src/hermes/model/merge/container.py +++ b/src/hermes/model/merge/container.py @@ -274,7 +274,7 @@ def __setitem__(self: Self, key: str, value: Union[JSON_LD_VALUE, BASIC_TYPE, TI self.prov_objects[2] = self.prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": f"software metadata after merge of values at {str(self.path+[key])}", - "schema:text": str(outer_most_parent.compact()), # TODO: maybe "prov:value" instead? + "schema:text": str(outer_most_parent.compact()), "prov:wasAttributedTo": self.prov_doc.get_hermes_command("process").ref, "prov:wasGeneratedBy": merge_activity.ref, "prov:wasDerivedFrom": {"@list": [self.prov_objects[1].ref, self.prov_objects[2].ref]}, From 2dcbb1a2cfe9664c59c199f35c4ed07e6dba7b1b Mon Sep 17 00:00:00 2001 From: "Kernchen, Sophie" Date: Tue, 1 Sep 2026 15:37:58 +0200 Subject: [PATCH 32/41] Fix HermesContext to HermesCacheManager --- src/hermes/commands/report/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hermes/commands/report/base.py b/src/hermes/commands/report/base.py index 1a9a88db..b4349645 100644 --- a/src/hermes/commands/report/base.py +++ b/src/hermes/commands/report/base.py @@ -235,7 +235,7 @@ def report_curate(self: Self) -> None: """ print("- Curate:") # load provenance data or error out - ctx = HermesContext() + ctx = HermesCacheManager() ctx.prepare_step("curate") with ctx["provenance"] as cache: try: @@ -306,7 +306,7 @@ def report_deposit(self: Self) -> None: """ print("- Deposit:") # load provenance data or error out - ctx = HermesContext() + ctx = HermesCacheManager() ctx.prepare_step("deposit") with ctx["provenance"] as cache: try: From c432df371ebbc730b780a6169f876596b2fe3665 Mon Sep 17 00:00:00 2001 From: "Kernchen, Sophie" Date: Tue, 1 Sep 2026 15:43:27 +0200 Subject: [PATCH 33/41] Flake8 --- docs/source/conf.py | 8 +++++++- src/hermes/commands/postprocess/base.py | 2 +- src/hermes/commands/postprocess/invenio.py | 1 - src/hermes/commands/postprocess/invenio_rdm.py | 1 - src/hermes/commands/process/base.py | 2 -- test/hermes_test/model/types/test_ld_container.py | 1 - 6 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index c627daba..fd30df54 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -28,6 +28,7 @@ sys.path.insert(0, os.path.abspath('../../src')) sys.path.append(os.path.abspath('_ext')) + def read_from_pyproject(file_path="../../pyproject.toml"): """ Reads the metadata from the pyproject.toml file. @@ -51,6 +52,7 @@ def read_from_pyproject(file_path="../../pyproject.toml"): except Exception as e: return f"An unexpected error occurred: {e}" + def read_authors_from_pyproject(): metadata = read_from_pyproject() authors = metadata.get("authors", []) @@ -59,6 +61,7 @@ def read_authors_from_pyproject(): # Convert the list of authors to a comma-separated string return ", ".join([author["name"] for author in authors]) + def read_version_from_pyproject(): metadata = read_from_pyproject() version = metadata.get("version", "") @@ -70,7 +73,8 @@ def read_version_from_pyproject(): # -- Project information ----------------------------------------------------- project = 'HERMES Workflow' -copyright = '2025 by Forschungszentrum Jülich (FZJ), German Aerospace Center (DLR) and Helmholtz-Zentrum Dresden-Rossendorf (HZDR)' +copyright = '2025 by Forschungszentrum Jülich (FZJ), German Aerospace Center (DLR)' \ + ' and Helmholtz-Zentrum Dresden-Rossendorf (HZDR)' author = read_authors_from_pyproject() # The full version, including alpha/beta/rc tags @@ -191,6 +195,7 @@ def read_version_from_pyproject(): # TODO: remove this workaround and remove "undoc-members" from autoapi_options once everything is documented # This removes all generated entries for known documented classes (because autoapi will add all attributes # it finds in the code no matter if they are described in a class doc string or not). + def autoapi_skip_member(app, obj_type, name, obj, skip, options): if obj_type == "attribute": if any(documented_type in obj.id for documented_type in [ @@ -201,5 +206,6 @@ def autoapi_skip_member(app, obj_type, name, obj, skip, options): return skip + def setup(app): app.connect("autoapi-skip-member", autoapi_skip_member) diff --git a/src/hermes/commands/postprocess/base.py b/src/hermes/commands/postprocess/base.py index 97b5a82c..27b8fd31 100644 --- a/src/hermes/commands/postprocess/base.py +++ b/src/hermes/commands/postprocess/base.py @@ -317,7 +317,7 @@ def __call__(self: Self, args: argparse.Namespace) -> None: if prov_doc is not None: # store provenance data - ctx = HermesCacheManager() + ctx = HermesCacheManager() ctx.prepare_step("postprocess") with ctx["provenance"] as cache: cache["result"] = prov_doc.ld_value diff --git a/src/hermes/commands/postprocess/invenio.py b/src/hermes/commands/postprocess/invenio.py index 5519aaf0..e2baa80c 100644 --- a/src/hermes/commands/postprocess/invenio.py +++ b/src/hermes/commands/postprocess/invenio.py @@ -13,7 +13,6 @@ import tomlkit from hermes.error import MisconfigurationError -from hermes.model.hermes_cache import HermesCacheManager from ..base import HermesCommand from .base import HermesPostprocessPlugin diff --git a/src/hermes/commands/postprocess/invenio_rdm.py b/src/hermes/commands/postprocess/invenio_rdm.py index 7e458b30..ff9fc549 100644 --- a/src/hermes/commands/postprocess/invenio_rdm.py +++ b/src/hermes/commands/postprocess/invenio_rdm.py @@ -11,7 +11,6 @@ import tomlkit from hermes.error import MisconfigurationError -from hermes.model.hermes_cache import HermesCacheManager from ..base import HermesCommand from .base import HermesPostprocessPlugin diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index 79163050..2c7f21ef 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -33,7 +33,6 @@ class HermesProcessPlugin(HermesPlugin): """ Base plugin that defines additional merge strategies. """ - def __call__(self: Self, command: "HermesProcessCommand") -> dict[Optional[str], dict[Optional[str], MergeAction]]: """ Execute the hermes process plugin `self`. @@ -314,7 +313,6 @@ def merge_data_from_harvesters( process_command = prov_doc.get_hermes_command("process") hermes_cache = prov_doc.get_hermes_cache() - # merge data from harvesters self.log.info("## Merge the metadata of the harvesters") ctx = HermesCacheManager() diff --git a/test/hermes_test/model/types/test_ld_container.py b/test/hermes_test/model/types/test_ld_container.py index a875ccec..9cf8f871 100644 --- a/test/hermes_test/model/types/test_ld_container.py +++ b/test/hermes_test/model/types/test_ld_container.py @@ -127,7 +127,6 @@ def test_to_native_python_datetime_value(self, mock_context): {"@value": "2022-02-22T00:00:00", "@type": "https://schema.org/DateTime"} ) == "2022-02-22T00:00:00" # TODO: #434 typed date is returned as string instead of date - def test_to_native_python_error(self, mock_context): cont = ld_container([{}], context=[mock_context]) with pytest.raises(TypeError): From 50186548fe04294f7094eae40b84a52837ad5b19 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Wed, 2 Sep 2026 11:35:22 +0200 Subject: [PATCH 34/41] adapt test for fix of 434 --- test/hermes_test/model/types/test_ld_container.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/hermes_test/model/types/test_ld_container.py b/test/hermes_test/model/types/test_ld_container.py index 9cf8f871..0442aef6 100644 --- a/test/hermes_test/model/types/test_ld_container.py +++ b/test/hermes_test/model/types/test_ld_container.py @@ -122,10 +122,12 @@ def test_to_native_python_basic_value(self, mock_context): def test_to_native_python_datetime_value(self, mock_context): cont = ld_container([{}], context=[mock_context]) - assert cont._to_native_python( + res = cont._to_native_python( "http://example.com/eggs", - {"@value": "2022-02-22T00:00:00", "@type": "https://schema.org/DateTime"} - ) == "2022-02-22T00:00:00" # TODO: #434 typed date is returned as string instead of date + {"@value": "2022-02-22T00:00:00", "@type": "http://schema.org/DateTime"} + ) + assert isinstance(res, datetime) + assert res == datetime.fromisoformat("2022-02-22T00:00:00") def test_to_native_python_error(self, mock_context): cont = ld_container([{}], context=[mock_context]) From 0ffe89370b8fef4447c546d604450a4f57aa6201 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Wed, 2 Sep 2026 12:48:35 +0200 Subject: [PATCH 35/41] remove __call__ method from invenio plugin --- src/hermes/commands/deposit/invenio.py | 57 +++++++++++--------------- 1 file changed, 24 insertions(+), 33 deletions(-) diff --git a/src/hermes/commands/deposit/invenio.py b/src/hermes/commands/deposit/invenio.py index c1fa5870..3eaefd84 100644 --- a/src/hermes/commands/deposit/invenio.py +++ b/src/hermes/commands/deposit/invenio.py @@ -266,39 +266,6 @@ def __init__(self) -> None: self.invenio_ctx = None - def __call__(self, command, *, client=None, resolver=None): - self.command = command - self.config = getattr(self.command.settings, self.platform_name) - - if client is None: - auth_token = self.config.auth_token - - # TODO reactivate this code again, once we use Zenodo OAuth again (once the refresh token works) - # If auth_token is a refresh-token, get the auth-token from that. - # if str(auth_token).startswith("REFRESH_TOKEN:"): - # _log.debug(f"Getting token from refresh_token {auth_token}") - # # TODO How do we know if this targets sandbox or not? - # # Now we assume it's sandbox - # connect_zenodo.setup(True) - # tokens = connect_zenodo.oauth_process() \ - # .get_tokens_from_refresh_token(auth_token.split("REFRESH_TOKEN:")[1]) - # _log.debug(f"Tokens: {str(tokens)}") - # auth_token = tokens.get("access_token", "") - # _log.debug(f"Auth Token: {auth_token}") - # # TODO Update the secret (github/lab token is needed) - - if not auth_token: - raise DepositionUnauthorizedError("No valid auth token given for deposition platform") - self.client = self.invenio_client_class(self.config, - auth_token=auth_token, platform_name=self.platform_name) - else: - self.client = client - - self.resolver = resolver or self.invenio_resolver_class(self.client) - self.links = {} - - super().__call__(command) - # TODO: Populate some data structure here? Or move more of this into __init__.py? def prepare(self) -> None: """Prepare the deposition on an Invenio-based platform. @@ -314,6 +281,30 @@ def prepare(self) -> None: - check whether required configuration options are present - update ``self.metadata`` with metadata collected during the checks """ + self.config = getattr(self.command.settings, self.platform_name) + + auth_token = self.config.auth_token + + # TODO reactivate this code again, once we use Zenodo OAuth again (once the refresh token works) + # If auth_token is a refresh-token, get the auth-token from that. + # if str(auth_token).startswith("REFRESH_TOKEN:"): + # _log.debug(f"Getting token from refresh_token {auth_token}") + # # TODO How do we know if this targets sandbox or not? + # # Now we assume it's sandbox + # connect_zenodo.setup(True) + # tokens = connect_zenodo.oauth_process() \ + # .get_tokens_from_refresh_token(auth_token.split("REFRESH_TOKEN:")[1]) + # _log.debug(f"Tokens: {str(tokens)}") + # auth_token = tokens.get("access_token", "") + # _log.debug(f"Auth Token: {auth_token}") + # # TODO Update the secret (github/lab token is needed) + + if not auth_token: + raise DepositionUnauthorizedError("No valid auth token given for deposition platform") + self.client = self.invenio_client_class(self.config, auth_token=auth_token, platform_name=self.platform_name) + + self.resolver = self.invenio_resolver_class(self.client) + self.links = {} conf_rec_id = self.config.record_id conf_doi = self.config.doi From e3b93b6c4c3bb8ba9d118d59a5e70468e5f5a59c Mon Sep 17 00:00:00 2001 From: "Kernchen, Sophie" Date: Wed, 2 Sep 2026 14:47:35 +0200 Subject: [PATCH 36/41] Fix config_invenio_rdm_record_id plugin and flake8 --- src/hermes/commands/postprocess/base.py | 2 +- src/hermes/commands/process/invenio_merge.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hermes/commands/postprocess/base.py b/src/hermes/commands/postprocess/base.py index 27b8fd31..1ab826db 100644 --- a/src/hermes/commands/postprocess/base.py +++ b/src/hermes/commands/postprocess/base.py @@ -264,7 +264,7 @@ def __call__(self: Self, args: argparse.Namespace) -> None: loaded_datas: list[ld_dict] = [] # add cache load operations to the provenance document for cache_load in cache_loads: - deposit_plugin = prov_doc.get_hermes_plugin("postprocess", cache_load[0]) + deposit_plugin = prov_doc.get_hermes_plugin("deposit", cache_load[0]) updated_metadata = prov_doc.shallow_search(lambda node: ( "prov:wasInfluencedBy" in node and node["prov:wasInfluencedBy"] == [deposit_plugin.ref] ))[0] diff --git a/src/hermes/commands/process/invenio_merge.py b/src/hermes/commands/process/invenio_merge.py index d36ee2da..518364c5 100644 --- a/src/hermes/commands/process/invenio_merge.py +++ b/src/hermes/commands/process/invenio_merge.py @@ -4,6 +4,7 @@ # SPDX-FileContributor: Michael Fritzsche +# flake8: noqa: C901 from typing import Union from typing_extensions import Self From 06d59d8a167eeae49b1d1f1933be3e6be314822e Mon Sep 17 00:00:00 2001 From: "Kernchen, Sophie" Date: Wed, 2 Sep 2026 15:18:33 +0200 Subject: [PATCH 37/41] Postprocess plugins for invenio_rdm --- pyproject.toml | 6 ++- .../commands/postprocess/invenio_rdm.py | 51 +++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c46fe270..6b62aa65 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,8 +69,10 @@ rodare = "hermes.commands.deposit.rodare:RodareDepositPlugin" [project.entry-points."hermes.postprocess"] config_invenio_record_id = "hermes.commands.postprocess.invenio:config_record_id" config_invenio_rdm_record_id = "hermes.commands.postprocess.invenio_rdm:config_record_id" -cff_doi = "hermes.commands.postprocess.invenio:cff_doi" -codemeta_doi = "hermes.commands.postprocess.invenio:codemeta_doi" +invenio_cff_doi = "hermes.commands.postprocess.invenio:cff_doi" +invenio_rdm_cff_doi = "hermes.commands.postprocess.invenio_rdm:cff_doi" +invenio_codemeta_doi = "hermes.commands.postprocess.invenio:codemeta_doi" +invenio_rdm_codemeta_doi = "hermes.commands.postprocess.invenio_rdm:codemeta_doi" [project.entry-points."hermes.process"] codemeta = "hermes.commands.process.standard_merge:CodemetaProcessPlugin" diff --git a/src/hermes/commands/postprocess/invenio_rdm.py b/src/hermes/commands/postprocess/invenio_rdm.py index ff9fc549..ce6b521d 100644 --- a/src/hermes/commands/postprocess/invenio_rdm.py +++ b/src/hermes/commands/postprocess/invenio_rdm.py @@ -6,10 +6,12 @@ # SPDX-FileContributor: Michael Fritzsche # SPDX-FileContributor: Stephan Druskat +import json import logging import tomlkit +from ruamel.yaml import YAML from hermes.error import MisconfigurationError from ..base import HermesCommand from .base import HermesPostprocessPlugin @@ -36,3 +38,52 @@ def __call__(self, command: HermesCommand): pass conf.setdefault("deposit", {}).setdefault("invenio_rdm", {})["record_id"] = deposition['record_id'] self.write(tomlkit.dump, conf, open('hermes.toml', 'w')) + + +class cff_doi(HermesPostprocessPlugin): + def __call__(self, command: HermesCommand): + + deposition = self.get_deposit_result("invenio_rdm") + + yaml = YAML() + yaml.default_flow_style = False + yaml.allow_unicode = True + yaml.indent(mapping=4, sequence=2, offset=0) + yaml.allow_unicode = True + + try: + cff = self.load(yaml.load, open('CITATION.cff', 'r')) + new_identifier = { + 'description': f"DOI for the published version {deposition['metadata']['version']} " + "[generated by hermes]", + 'type': 'doi', + 'value': deposition['metadata']['prereserve_doi']['doi'] + } + if 'identifiers' in cff: + cff['identifiers'].append(new_identifier) + else: + cff['identifiers'] = [new_identifier] + self.write(yaml.dump, cff, open('CITATION.cff', 'w')) + except Exception as e: + raise RuntimeError("Update of CITATION.cff failed.") from e + + +class codemeta_doi(HermesPostprocessPlugin): + def __call__(self, command: HermesCommand): + deposition = self.get_deposit_result("invenio_rdm") + doi = deposition['metadata']['prereserve_doi']['doi'] + try: + with open("codemeta.json", "r") as file: + codemeta = self.load(json.load, file) + if "@id" not in codemeta: + codemeta["@id"] = doi + if "referencePublication" not in codemeta: + codemeta["referencePublication"] = doi + elif isinstance(codemeta["referencePublication"], list): + codemeta["referencePublication"].append(doi) + else: + codemeta["referencePublication"] = [codemeta["referencePublication"], doi] + with open("codemeta.json", "w") as file: + self.write(json.dump, codemeta, file) + except Exception as e: + raise RuntimeError("Update of CITATION.cff failed.") from e \ No newline at end of file From 7e7f466b9d796b2f7e9969bf5638316e5e360c98 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Wed, 2 Sep 2026 15:42:59 +0200 Subject: [PATCH 38/41] fix report command and flake8 --- .../commands/postprocess/invenio_rdm.py | 2 +- src/hermes/commands/report/base.py | 28 ++++++++++--------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/hermes/commands/postprocess/invenio_rdm.py b/src/hermes/commands/postprocess/invenio_rdm.py index ce6b521d..ce3c6b73 100644 --- a/src/hermes/commands/postprocess/invenio_rdm.py +++ b/src/hermes/commands/postprocess/invenio_rdm.py @@ -86,4 +86,4 @@ def __call__(self, command: HermesCommand): with open("codemeta.json", "w") as file: self.write(json.dump, codemeta, file) except Exception as e: - raise RuntimeError("Update of CITATION.cff failed.") from e \ No newline at end of file + raise RuntimeError("Update of CITATION.cff failed.") from e diff --git a/src/hermes/commands/report/base.py b/src/hermes/commands/report/base.py index b4349645..7851c51e 100644 --- a/src/hermes/commands/report/base.py +++ b/src/hermes/commands/report/base.py @@ -417,7 +417,7 @@ def report_postprocess(self: Self) -> None: "prov:actedOnBehalfOf" in node and node["prov:actedOnBehalfOf"] == [base_plugin.ref] ))[0] print( - f" - Plugin used:\n - {plugin['@id'][28]} ({plugin['schema:name'][0]}, version " + f" - Plugin used:\n - {plugin['@id'][28:]} ({plugin['schema:name'][0]}, version " f"{vers if (vers := plugin.get('schema:softwareVersion', False)) else 'N/A'})" ) cache_loads = prov_doc.shallow_search(lambda node: ( @@ -428,11 +428,11 @@ def report_postprocess(self: Self) -> None: print(" - Used deposit results:") for index, cache_load in enumerate(cache_loads, start=1): source_id = cache_load["prov:used"][0]["@id"] - source = prov_doc.shallow_search(lambda node: ("@id" in node and node["@id"] == [source_id]))[0] + source = prov_doc.shallow_search(lambda node: ("@id" in node and node["@id"] == source_id))[0] print( - f" - Load {index} at {cache_load['prov:startedAtTime']} took " - f"{cache_load['prov:endedAtTime']-cache_load['prov:startedAtTime']} from:\n" - f" - {source['schema:url']}" + f" - Load {index} at {cache_load['prov:startedAtTime'][0]} took " + f"{cache_load['prov:endedAtTime'][0]-cache_load['prov:startedAtTime'][0]} from:\n" + f" - {source['schema:url'][0]}" ) io_ops = prov_doc.shallow_search(lambda node: ( "prov:wasAssociatedWith" in node and @@ -442,7 +442,9 @@ def report_postprocess(self: Self) -> None: loads, writes = [], [] for io_op in io_ops: used = io_op["prov:used"][0]["@id"] - if "prov:wasDerivedFrom" in prov_doc.shallow_search(lambda node: ("@id" in node and node["@id"] == used)): + if "prov:wasDerivedFrom" in prov_doc.shallow_search( + lambda node: ("@id" in node and node["@id"] == used) + )[0]: writes.append(io_op) else: loads.append(io_op) @@ -450,11 +452,11 @@ def report_postprocess(self: Self) -> None: print(" - Loaded data from:") for index, load in enumerate(loads): source_id = load["prov:used"][0]["@id"] - source = prov_doc.shallow_search(lambda node: ("@id" in node and node["@id"] == [source_id]))[0] + source = prov_doc.shallow_search(lambda node: ("@id" in node and node["@id"] == source_id))[0] print( - f" - Load {index} at {load['prov:startedAtTime']} took " - f"{load['prov:endedAtTime']-load['prov:startedAtTime']} from:\n" - f" - {source['schema:url']}" + f" - Load {index} at {load['prov:startedAtTime'][0]} took " + f"{load['prov:endedAtTime'][0]-load['prov:startedAtTime'][0]} from:\n" + f" - {source['schema:url'][0]}" ) # print info on general writes of the plugin print(" - Written data to:") @@ -463,7 +465,7 @@ def report_postprocess(self: Self) -> None: "prov:wasGeneratedBy" in node and node["prov:wasGeneratedBy"] == [write.ref] ))[0] print( - f" - Load {index} at {write['prov:startedAtTime']} took " - f"{write['prov:endedAtTime']-write['prov:startedAtTime']} from:\n" - f" - {target['schema:url']}" + f" - Write {index} at {write['prov:startedAtTime'][0]} took " + f"{write['prov:endedAtTime'][0]-write['prov:startedAtTime'][0]} from:\n" + f" - {target['schema:url'][0]}" ) From e9dd64748d9ff42e894597f9f0d622c4c73b218a Mon Sep 17 00:00:00 2001 From: "Kernchen, Sophie" Date: Wed, 2 Sep 2026 16:00:47 +0200 Subject: [PATCH 39/41] Correct test with new plugin name --- .../commands/postprocess/test_invenio_postprocess.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/hermes_test/commands/postprocess/test_invenio_postprocess.py b/test/hermes_test/commands/postprocess/test_invenio_postprocess.py index 3a2a77fc..7150d15f 100644 --- a/test/hermes_test/commands/postprocess/test_invenio_postprocess.py +++ b/test/hermes_test/commands/postprocess/test_invenio_postprocess.py @@ -58,7 +58,7 @@ def test_invenio_postprocess(tmp_path, monkeypatch): communities = "api/communities" [postprocess] -run = ["config_invenio_record_id", "cff_doi", "codemeta_doi"] +run = ["config_invenio_record_id", "invenio_cff_doi", "invenio_codemeta_doi"] """ ) @@ -101,7 +101,7 @@ def test_invenio_postprocess(tmp_path, monkeypatch): communities = "api/communities" [postprocess] -run = ["config_invenio_record_id", "cff_doi", "codemeta_doi"] +run = ["config_invenio_record_id", "invenio_cff_doi", "invenio_codemeta_doi"] """ ).unwrap() assert result_cff == yaml.YAML().load( From c9e50fda8e996fb8de7ba2d387b6887ba82dce70 Mon Sep 17 00:00:00 2001 From: "Kernchen, Sophie" Date: Thu, 3 Sep 2026 00:22:40 +0200 Subject: [PATCH 40/41] Remove unused variables --- src/hermes/commands/base.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/hermes/commands/base.py b/src/hermes/commands/base.py index 815eb63c..b808d152 100644 --- a/src/hermes/commands/base.py +++ b/src/hermes/commands/base.py @@ -28,8 +28,6 @@ class HermesSettings(BaseSettings): model_config = SettingsConfigDict(env_file_encoding='utf-8') - logging: dict = {} # FIXME: Is this still used? Even if removed, no tests fail... - class HermesCommand(abc.ABC): """Base class for a HERMES workflow command. @@ -57,7 +55,6 @@ def __init__(self: Self, parser: argparse.ArgumentParser) -> None: self.settings = None self.log = logging.getLogger(f"hermes.{self.command_name}") - self.errors = [] # FIXME: not used, right? def init_plugins(self: Self) -> dict[str, type["HermesPlugin"]]: """ From 6c14982c6ffd4e1ce82a484d9d95be1301e26b3e Mon Sep 17 00:00:00 2001 From: "Kernchen, Sophie" Date: Thu, 3 Sep 2026 00:28:30 +0200 Subject: [PATCH 41/41] Remove outdated parameter description --- src/hermes/commands/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hermes/commands/base.py b/src/hermes/commands/base.py index b808d152..889eaf67 100644 --- a/src/hermes/commands/base.py +++ b/src/hermes/commands/base.py @@ -23,7 +23,6 @@ class HermesSettings(BaseSettings): Attributes: model_config (SettingsConfigDict): The settings config dict for the settings of hermes. - logging (dict): ... """ model_config = SettingsConfigDict(env_file_encoding='utf-8')