Skip to content

Commit c86b22a

Browse files
georgeRobertsongeorgeRobertson
committed
feat: add empty_entity check post filters for mandatory entities
1 parent 3eca2f1 commit c86b22a

8 files changed

Lines changed: 181 additions & 25 deletions

File tree

‎docs/advanced_guidance/json_schemas/entity_relationships.schema.json‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@
1717
"type": "string"
1818
}
1919
},
20+
"is_root_entity": {
21+
"type": "boolean"
22+
},
2023
"mandatory": {
2124
"type": "boolean"
2225
},
@@ -31,12 +34,16 @@
3134
},
3235
"no_valid_records_error_message": {
3336
"type": "string"
37+
},
38+
"empty_entity_error_code": {
39+
"type": "string",
40+
"minLength": 1
41+
},
42+
"empty_entity_error_message": {
43+
"type": "string",
44+
"minLength": 1
3445
}
3546
},
36-
"required": [
37-
"parent_entity",
38-
"join_fields"
39-
],
4047
"additionalProperties": false
4148
}
4249
}

‎src/dve/core_engine/configuration/v1/__init__.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,10 @@ class _LinkageConfig(BaseModel):
114114
"Records removed due to no valid parent record"
115115
)
116116
"""The error code to emit if the entity contains records that are orphaned by parent record rejections""" # pylint: disable=C0301
117+
empty_entity_error_code: ErrorCode = "EmptyEntity"
118+
"""The error code to emit if a mandatory entity has no valid remaining records"""
119+
empty_entity_error_message: ErrorMessage = "no valid records remaining"
120+
"""The error message to emit if a mandatory entity has no valid remaining records"""
117121

118122
@model_validator(mode="after")
119123
def _check_root_no_parent_or_join_keys(self):
@@ -123,6 +127,17 @@ def _check_root_no_parent_or_join_keys(self):
123127
)
124128
return self
125129

130+
@model_validator(mode="after")
131+
def _check_non_root_entities_have_a_defined_parent(self):
132+
"""Check that non root entities have a parent defined."""
133+
if not self.is_root_entity and self.parent_entity is None:
134+
raise ValueError(
135+
'Non-root entity has no defined parent entity. If you intend this to be a root ' \
136+
'entity you must specify `"is_root_entity": true` for the entity. ' \
137+
'Otherwise you must specify a `"parent_entity": "<EntityName>"` for this entity.'
138+
)
139+
return self
140+
126141
@model_validator(mode="after")
127142
def _check_root_mandatory(self):
128143
if self.is_root_entity and not self.mandatory:

‎src/dve/core_engine/configuration/v1/hierarchy.py‎

Lines changed: 44 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
import json
44
from typing import Any, Iterable, Optional, Union
55

6-
from pydantic import BaseModel, Field
6+
from pydantic import BaseModel, Field, model_validator
77

88
from dve.core_engine.configuration.v1 import V1EngineConfig, _LinkageConfig
99
from dve.core_engine.type_hints import EntityName, ErrorCode, ErrorMessage
@@ -26,6 +26,19 @@ class HierarchyNode(BaseModel):
2626
missing_parent_id_error_message: Optional[ErrorMessage] = (
2727
"Records removed due to no valid parent record"
2828
)
29+
empty_entity_error_code: ErrorCode = "EmptyEntity"
30+
empty_entity_error_message: ErrorMessage = "no valid records remaining"
31+
32+
@model_validator(mode="after")
33+
def validate_empty_error_details(self):
34+
"""
35+
Removes the default messaging for checking empty entities as not performed on
36+
non mandatory nodes/entities
37+
"""
38+
if not self.mandatory:
39+
self.empty_entity_error_code = None
40+
self.empty_entity_error_message = None
41+
return self
2942

3043
def get_descendents(self) -> list["HierarchyNode"]:
3144
"""Recursively list all descendents of the node"""
@@ -129,12 +142,15 @@ def determine_trees(
129142

130143
for name, linkage_detail in entity_relationships.items():
131144
for main_entity, parent_node in top_level_parents.items():
145+
if linkage_detail.is_root_entity:
146+
break
147+
132148
if (
133149
linkage_detail.parent_entity == main_entity
134150
or linkage_detail.parent_entity in parent_node.get_descendent_names()
135151
):
136152
parent_node.add_child_node(
137-
linkage_detail.parent_entity,
153+
linkage_detail.parent_entity, # type: ignore
138154
HierarchyNode(entity_name=name, **linkage_detail.model_dump()),
139155
)
140156
break
@@ -166,3 +182,29 @@ def from_engine_config(cls, engine_config: V1EngineConfig):
166182
entity_relationships=engine_config.entity_relationships,
167183
)
168184
)
185+
186+
def get_all_mandatory_nodes(
187+
self,
188+
node: Optional[HierarchyNode] = None,
189+
mandatory_nodes: Optional[list[HierarchyNode]] = None,
190+
nodes_visited: Optional[set[EntityName]] = None,
191+
) -> list[HierarchyNode]:
192+
"""Find and return all mandatory nodes"""
193+
if mandatory_nodes is None:
194+
mandatory_nodes = []
195+
196+
if nodes_visited is None:
197+
nodes_visited = set()
198+
199+
if node is None:
200+
for _node in self.entity_trees.values():
201+
self.get_all_mandatory_nodes(_node, mandatory_nodes, nodes_visited)
202+
203+
if node:
204+
if node.mandatory and node.entity_name not in nodes_visited:
205+
nodes_visited.add(node.entity_name)
206+
mandatory_nodes.append(node)
207+
for child_node in node.children:
208+
self.get_all_mandatory_nodes(child_node, mandatory_nodes, nodes_visited)
209+
210+
return mandatory_nodes

‎src/dve/core_engine/type_hints.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,13 @@
134134
FieldValue = Optional[Any]
135135
"""The value that caused the error."""
136136
ErrorCategory = Literal[
137-
"Blank", "Wrong format", "Bad value", "Bad file", "Parent Missing", "Children missing"
137+
"Blank",
138+
"Wrong format",
139+
"Bad value",
140+
"Bad file",
141+
"Parent Missing",
142+
"Children missing",
143+
"Empty entity",
138144
]
139145
"""A string indicating the category of the error."""
140146
RecordIndex = Optional[int]

‎src/dve/pipeline/pipeline.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
import dve.reporting.excel_report as er
2020
from dve.common.error_utils import (
21+
BackgroundMessageWriter,
2122
dump_feedback_errors,
2223
dump_processing_errors,
2324
get_feedback_errors_uri,
@@ -545,6 +546,44 @@ def data_contract_step(
545546

546547
return processed_files, failed_processing
547548

549+
def check_mandatory_entities_have_records(
550+
self,
551+
working_directory: URI,
552+
entities: EntityManager,
553+
entity_hierarchy: EntityHierarchy,
554+
key_fields: Optional[dict[str, list[str]]] = None,
555+
) -> None:
556+
"""
557+
Check that mandatory entities have at least one record post business rules. Otherwise,
558+
raise a submission rejection error message.
559+
"""
560+
with BackgroundMessageWriter(
561+
working_directory=working_directory,
562+
dve_stage="business_rules",
563+
key_fields=key_fields,
564+
logger=self._logger,
565+
) as msg_writer:
566+
_msgs = []
567+
for node in entity_hierarchy.get_all_mandatory_nodes():
568+
entity_name = node.entity_name
569+
if node.mandatory and self.get_entity_count(entities[entity_name]) == 0:
570+
self._logger.info(
571+
f"Found 0 records in mandatory entity {entity_name} after applying all business rules" # pylint: disable=C0301
572+
)
573+
_msgs.append(
574+
FeedbackMessage(
575+
entity=entity_name,
576+
record=None,
577+
error_location=entity_name,
578+
error_message=node.empty_entity_error_message,
579+
failure_type="submission",
580+
error_type="submission",
581+
error_code=node.empty_entity_error_code,
582+
category="Empty entity",
583+
)
584+
)
585+
msg_writer.write_queue.put(_msgs)
586+
548587
def apply_business_rules( # pylint: disable=R0914,R0915
549588
self, submission_info: SubmissionInfo, submission_status: Optional[SubmissionStatus] = None
550589
) -> tuple[SubmissionInfo, SubmissionStatus]:
@@ -725,6 +764,10 @@ def apply_business_rules( # pylint: disable=R0914,R0915
725764
)
726765
)
727766

767+
self.check_mandatory_entities_have_records(
768+
working_directory, entity_manager, entity_hierarchy
769+
)
770+
728771
submission_status.number_of_records = self.get_entity_count(
729772
entity=entity_manager.entities[f"""Original{rules.global_variables.get(
730773
'entity',

‎tests/features/flights.feature‎

Lines changed: 24 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -56,11 +56,14 @@ Feature: Pipeline tests using the flights dataset
5656
| record | CountryIdIsMissing | 1 |
5757
When I run the business rules phase
5858
Then there are errors with the following details and associated error_count from the business_rules phase
59-
| ErrorType | ErrorCode | error_count |
60-
| record | AirportHasNoCountry | 3 |
61-
| record | StaffHasNoAirport | 15 |
62-
| record | FlightHasNoAirport | 10 |
63-
| record | PassengerHasNoFlight | 25 |
59+
| ErrorType | ErrorCode | error_count |
60+
| record | AirportHasNoCountry | 3 |
61+
| record | StaffHasNoAirport | 15 |
62+
| record | FlightHasNoAirport | 10 |
63+
| record | PassengerHasNoFlight | 25 |
64+
| submission | NoValidCountries | 1 |
65+
| submission | NoValidAirports | 1 |
66+
| submission | NoValidStaff | 1 |
6467
And the final entities have the following row counts
6568
| entity_name | row_count |
6669
| country | 0 |
@@ -73,7 +76,7 @@ Feature: Pipeline tests using the flights dataset
7376
And The statistics entry for the submission shows the following information
7477
| parameter | value |
7578
| record_count | 1 |
76-
| number_submission_rejections | 0 |
79+
| number_submission_rejections | 3 |
7780
| number_record_rejections | 54 |
7881
| number_warnings | 0 |
7982

@@ -129,8 +132,11 @@ Feature: Pipeline tests using the flights dataset
129132
And there are no record rejections from the data_contract phase
130133
When I run the business rules phase
131134
Then there are errors with the following details and associated error_count from the business_rules phase
132-
| ErrorType | ErrorCode | error_count |
133-
| record | CountryHasNoAirport | 1 |
135+
| ErrorType | ErrorCode | error_count |
136+
| record | CountryHasNoAirport | 1 |
137+
| submission | NoValidCountries | 1 |
138+
| submission | NoValidAirports | 1 |
139+
| submission | NoValidStaff | 1 |
134140
And the final entities have the following row counts
135141
| entity_name | row_count |
136142
| country | 0 |
@@ -143,7 +149,7 @@ Feature: Pipeline tests using the flights dataset
143149
And The statistics entry for the submission shows the following information
144150
| parameter | value |
145151
| record_count | 1 |
146-
| number_submission_rejections | 0 |
152+
| number_submission_rejections | 3 |
147153
| number_record_rejections | 1 |
148154
| number_warnings | 0 |
149155

@@ -163,11 +169,14 @@ Feature: Pipeline tests using the flights dataset
163169
And there are no record rejections from the data_contract phase
164170
When I run the business rules phase
165171
Then there are errors with the following details and associated error_count from the business_rules phase
166-
| ErrorType | Status | ErrorCode | error_count |
167-
| record | error | InvalidFlightDestination | 2 |
168-
| record | error | PassengerHasNoFlight | 4 |
169-
| record | error | AirportHasNoStaff | 1 |
170-
| record | error | CountryHasNoAirport | 1 |
172+
| ErrorType | Status | ErrorCode | error_count |
173+
| record | error | InvalidFlightDestination | 2 |
174+
| record | error | PassengerHasNoFlight | 4 |
175+
| record | error | AirportHasNoStaff | 1 |
176+
| record | error | CountryHasNoAirport | 1 |
177+
| submission | error | NoValidCountries | 1 |
178+
| submission | error | NoValidAirports | 1 |
179+
| submission | error | NoValidStaff | 1 |
171180
And the final entities have the following row counts
172181
| entity_name | row_count |
173182
| country | 0 |
@@ -180,7 +189,7 @@ Feature: Pipeline tests using the flights dataset
180189
And The statistics entry for the submission shows the following information
181190
| parameter | value |
182191
| record_count | 1 |
183-
| number_submission_rejections | 0 |
192+
| number_submission_rejections | 3 |
184193
| number_record_rejections | 8 |
185194
| number_warnings | 0 |
186195

‎tests/test_core_engine/test_hierarchy.py‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,8 @@ def test_linkage_config_load():
269269
"no_valid_records_error_message": "parent record removed as no valid child records",
270270
"missing_parent_id_error_code": null,
271271
"missing_parent_id_error_message": null,
272+
"empty_entity_error_code": null,
273+
"empty_entity_error_message": null,
272274
"children": {
273275
"ds_003": {
274276
"parent_entity": "ds_001",
@@ -280,6 +282,8 @@ def test_linkage_config_load():
280282
"no_valid_records_error_message": "parent record removed as no valid child records",
281283
"missing_parent_id_error_code": "DS003NoParent",
282284
"missing_parent_id_error_message": "record removed as no parent",
285+
"empty_entity_error_code": null,
286+
"empty_entity_error_message": null,
283287
"children": {}
284288
},
285289
"ds_101": {
@@ -292,6 +296,8 @@ def test_linkage_config_load():
292296
"no_valid_records_error_message": "{{ ds_001_id }} removed as no valid ds_101 records",
293297
"missing_parent_id_error_code": "DS101NoParent",
294298
"missing_parent_id_error_message": "record removed as no parent",
299+
"empty_entity_error_code": null,
300+
"empty_entity_error_message": null,
295301
"children": {
296302
"ds_201": {
297303
"parent_entity": "ds_101",
@@ -303,6 +309,8 @@ def test_linkage_config_load():
303309
"no_valid_records_error_message": "parent record removed as no valid child records",
304310
"missing_parent_id_error_code": "DS201NoParent",
305311
"missing_parent_id_error_message": "record removed as no parent",
312+
"empty_entity_error_code": null,
313+
"empty_entity_error_message": null,
306314
"children": {
307315
"ds_202": {
308316
"parent_entity": "ds_201",
@@ -314,6 +322,8 @@ def test_linkage_config_load():
314322
"no_valid_records_error_message": "parent record removed as no valid child records",
315323
"missing_parent_id_error_code": "MissingParentRecord",
316324
"missing_parent_id_error_message": "Records removed due to no valid parent record",
325+
"empty_entity_error_code": "EmptyEntity",
326+
"empty_entity_error_message": "no valid records remaining",
317327
"children": {}
318328
}
319329
}
@@ -341,6 +351,8 @@ def test_linkage_config_load():
341351
"no_valid_records_error_message": "{{ ds_001_id }} removed as no valid ds_101 records",
342352
"missing_parent_id_error_code": "DS101NoParent",
343353
"missing_parent_id_error_message": "record removed as no parent",
354+
"empty_entity_error_code": null,
355+
"empty_entity_error_message": null,
344356
"children": {
345357
"ds_201": {
346358
"parent_entity": "ds_101",
@@ -352,6 +364,8 @@ def test_linkage_config_load():
352364
"no_valid_records_error_message": "parent record removed as no valid child records",
353365
"missing_parent_id_error_code": "DS201NoParent",
354366
"missing_parent_id_error_message": "record removed as no parent",
367+
"empty_entity_error_code": null,
368+
"empty_entity_error_message": null,
355369
"children": {
356370
"ds_202": {
357371
"parent_entity": "ds_201",
@@ -363,10 +377,20 @@ def test_linkage_config_load():
363377
"no_valid_records_error_message": "parent record removed as no valid child records",
364378
"missing_parent_id_error_code": "MissingParentRecord",
365379
"missing_parent_id_error_message": "Records removed due to no valid parent record",
380+
"empty_entity_error_code": "EmptyEntity",
381+
"empty_entity_error_message": "no valid records remaining",
366382
"children": {}
367383
}
368384
}
369385
}
370386
}
371387
}""")
372-
388+
389+
390+
def test_get_all_mandatory_nodes():
391+
with NamedTemporaryFile("w") as tmp:
392+
tmp.write(CONFIG_WITH_LINKAGE)
393+
tmp.flush()
394+
hierarchy = EntityHierarchy.from_dischema(tmp.name)
395+
396+
assert len(hierarchy.get_all_mandatory_nodes()) == 1

0 commit comments

Comments
 (0)