feat: following review, tweak location of additional reader checks config and add json schema support

Author: stevenhsd

docs/advanced_guidance/json_schemas/contract/components/base_entity.schema.json

@@ -26,8 +26,15 @@
             "items": {
                 "type": "string"
             }
+        },
+        "reader_additional_checks": {
+            "description": "A mapping of additional checks to perform on entities after initial read",
+            "type": "object",
+            "additionalProperties": {
+                "$ref": "reader_additional_checks.json"
         }
-    },
+    }
+},
 
     "required": ["fields"]
 }

docs/advanced_guidance/json_schemas/contract/components/reader_additional_checks.schema.json

@@ -0,0 +1,22 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "data-ingest:contract/components/reader_additional_checks.schema.json",
+    "title": "reader_additional_checks",
+    "description": "Additional checks to perform on initially read entities",
+    "type": "object",
+    "properties": {
+        "error_code": {
+            "description": "The code to be used for the additional check specified",
+            "type": "string"
+        },
+        "error_message": {
+            "description": "The message to be displayed for the additional check specified.",
+            "type": "string"
+        }
+    },
+    "required": [
+        "error_code",
+        "error_message"
+    ],
+    "additionalProperties": false
+}

src/dve/core_engine/backends/base/reader.py

@@ -10,7 +10,10 @@
 
 from dve.core_engine.backends.exceptions import MessageBearingError, ReaderLacksEntityTypeSupport
 from dve.core_engine.backends.types import EntityName, EntityType
-from dve.core_engine.configuration.v1 import AllowedAdditionalChecks, _ReaderAdditionalChecksConfig
+from dve.core_engine.configuration.v1 import (
+    AllowedAdditionalReaderChecks,
+    _ReaderAdditionalChecksConfig,
+)
 from dve.core_engine.message import FeedbackMessage
 from dve.core_engine.type_hints import URI, ArbitraryFunction, WrapDecorator
 from dve.parser.file_handling.service import open_stream
@@ -111,7 +114,7 @@ def read_to_entity_type(
         schema: type[BaseModel],
         all_model_fields: Optional[set[str]] = None,
         additional_checks: Optional[
-            dict[AllowedAdditionalChecks, _ReaderAdditionalChecksConfig]
+            dict[AllowedAdditionalReaderChecks, _ReaderAdditionalChecksConfig]
         ] = None,
     ):
         """Read to the specified entity type, if supported.

src/dve/core_engine/configuration/v1/__init__.py

@@ -57,7 +57,7 @@
 """The operation """
 RuleType = type[AbstractStep]
 """The metadata step type implemented by the rule."""
-AllowedAdditionalChecks = Literal["check_empty"]
+AllowedAdditionalReaderChecks = Literal["check_empty"]
 """Additional checks to be performed in the file_transformation stage"""
 
 
@@ -162,10 +162,6 @@ class _ReaderConfig(BaseModel):  # type: ignore
     """Keyword arguments for the reader."""
     field_names: Optional[list[str]] = None
     """The field names to request from the reader. These are deprecated and will not be used."""
-    additional_checks: dict[AllowedAdditionalChecks, _ReaderAdditionalChecksConfig] = Field(
-        default_factory=dict
-    )
-    """Additional checks to be performed after the entity is read"""
 
 
 class _ModelConfig(_SchemaConfig):
@@ -177,6 +173,10 @@ class _ModelConfig(_SchemaConfig):
     """A single key field to be used by the model."""
     reader_config: dict[Extension, _ReaderConfig]
     """Reader configuration options for the model."""
+    reader_additional_checks: dict[AllowedAdditionalReaderChecks, _ReaderAdditionalChecksConfig] = (
+        Field(default_factory=dict)
+    )
+    """Additional checks to be performed after the entity is read"""
     aliases: dict[FieldName, FieldName] = Field(default_factory=dict)
     """An alias field name mapping."""
 

src/dve/pipeline/pipeline.py

@@ -238,7 +238,7 @@ def write_file_to_parquet(
                             model_name,
                             stringify_model(model),  # type: ignore
                             get_all_model_fields(models.values()),  # type: ignore
-                            dataset[model_name].reader_config[f".{ext.lower()}"].additional_checks,
+                            dataset[model_name].reader_additional_checks,
                         ),
                         f"{out}{model_name}",
                     )

tests/testdata/flights/flights_add_reader_checks.dischema.json

@@ -44,15 +44,15 @@
                         "kwargs": {
                             "record_tag": "airport",
                             "root_tag": "country"
-                        },
-                        "additional_checks": {
-                            "check_empty": {
-                                "error_code": "AIRPORTEMPTY",
-                                "error_message": "No airport records included in submission"
-                        }
                         }
                     }
                 },
+                "reader_additional_checks": {
+                    "check_empty": {
+                        "error_code": "AIRPORTEMPTY",
+                        "error_message": "No airport records included in submission"
+                    }
+                },               
                 "key_field": "airport_id",
                 "mandatory_fields": [
                     "airport_id"