From 84607e1f51de0970f9c68627f26e3603dfd45cf0 Mon Sep 17 00:00:00 2001 From: Mathias Hansen Date: Mon, 5 Oct 2026 15:14:30 +0200 Subject: [PATCH] Add a raw property with the unmapped source columns to every output feature Each output feature gets properties.raw, a JSON object of the source row's columns, so downstream can map any field without further fork changes. Columns the conform maps directly (a column name, or a list of column names, for any SCHEMA key or lat/lon) are left out because their values are already output properties. Columns used only inside conform functions stay, with their original pre-function values. Columns starting with oa: and empty values are skipped. The ESRI cache now writes every field from the layer metadata instead of only the fields the conform names. Conform fields missing from the metadata are still written as empty columns and logged once. Claude-Session: https://claude.ai/code/session_01M9dZ4UWYaMwr9e9cwwKpyS --- openaddr/cache.py | 10 ++++++---- openaddr/conform.py | 19 +++++++++++++++++++ openaddr/tests/__init__.py | 10 +++++----- openaddr/tests/conform.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 9 deletions(-) diff --git a/openaddr/cache.py b/openaddr/cache.py index c577aadb..c3c74916 100644 --- a/openaddr/cache.py +++ b/openaddr/cache.py @@ -403,10 +403,12 @@ def download(self, source_urls, workdir, source_config): metadata = downloader.get_metadata() - if query_fields is None: - field_names = [f['name'] for f in metadata['fields']] - else: - field_names = query_fields[:] + field_names = [f['name'] for f in metadata['fields']] + + missing_fields = [f for f in query_fields or [] if f not in field_names] + if missing_fields: + _L.warning("Conform fields missing from layer metadata: %s", missing_fields) + field_names += missing_fields if GEOM_FIELDNAME not in field_names: field_names.append(GEOM_FIELDNAME) diff --git a/openaddr/conform.py b/openaddr/conform.py index 92707df7..efd3d6db 100644 --- a/openaddr/conform.py +++ b/openaddr/conform.py @@ -1012,6 +1012,8 @@ def row_transform_and_convert(source_config, row): c = source_config.data_source["conform"] + raw = row_raw_columns(source_config, row) + "Attribute tags can utilize processing fxns" for k, v in c.items(): if k in source_config.SCHEMA and isinstance(v, list): @@ -1035,11 +1037,28 @@ def row_transform_and_convert(source_config, row): if source_config.layer == "addresses": feat['properties'] = row_canonicalize_unit_and_number(source_config.data_source, feat['properties']) + feat['properties']['raw'] = raw + if feat['geometry'] and len(feat['geometry']['coordinates']) > 0: feat['geometry']['coordinates'] = set_precision(feat['geometry']['coordinates'], 7) return feat +def row_raw_columns(sc, row): + """Copy the source columns that are not already in the output, before any conform function runs. + Columns mapped directly (a column name, or a list of column names) are excluded because their + values are output properties. Columns used only inside functions stay.""" + c = sc.data_source["conform"] + mapped = set() + for key in list(sc.SCHEMA) + ['lat', 'lon']: + v = c.get(key) + if isinstance(v, str): + mapped.add(v) + elif isinstance(v, list): + mapped.update(f for f in v if isinstance(f, str)) + return {k: v for k, v in row.items() + if k not in mapped and not k.lower().startswith('oa:') and v not in (None, '')} + def row_merge(sc, row, key): "Merge multiple columns like 'Maple','St' to 'Maple St'" merge_data = [(row.get(field, '') or '').strip() for field in sc.data_source["conform"][key]] diff --git a/openaddr/tests/__init__.py b/openaddr/tests/__init__.py index 5dfecc58..56621043 100644 --- a/openaddr/tests/__init__.py +++ b/openaddr/tests/__init__.py @@ -528,7 +528,7 @@ def test_single_car(self): state = dict(zip(*json.load(file))) self.assertIsNotNone(state['cache']) - self.assertEqual(state['fingerprint'], '4a8047f90dbfe176c2a2b148837dae36') + self.assertEqual(state['fingerprint'], '4bc0e7aa15aed0d183d0485b964fdd4c') self.assertIsNotNone(state['processed']) self.assertIsNotNone(state['preview']) self.assertIsNotNone(state['pmtiles']) @@ -557,7 +557,7 @@ def test_single_car_cached(self): state = dict(zip(*json.load(file))) self.assertIsNotNone(state['cache']) - self.assertEqual(state['fingerprint'], '056bdaab3334e709bf29b0b2f1fcf8c4') + self.assertEqual(state['fingerprint'], '4bc0e7aa15aed0d183d0485b964fdd4c') self.assertIsNotNone(state['processed']) self.assertIsNone(state['preview']) @@ -576,7 +576,7 @@ def test_single_car_old_cached(self): state = dict(zip(*json.load(file))) self.assertIsNotNone(state['cache']) - self.assertEqual(state['fingerprint'], '056bdaab3334e709bf29b0b2f1fcf8c4') + self.assertEqual(state['fingerprint'], '4bc0e7aa15aed0d183d0485b964fdd4c') self.assertIsNotNone(state['processed']) self.assertIsNone(state['preview']) @@ -1019,7 +1019,7 @@ def test_single_tx_waco(self): self.assertEqual(rows[0]['properties']['region'], u'TX') self.assertEqual(rows[0]['properties']['id'], u'') self.assertEqual(rows[0]['properties']['number'], u'308') - self.assertEqual(rows[0]['properties']['hash'], u'd9db73ddf7ed0e2b') + self.assertEqual(rows[0]['properties']['hash'], u'2f31420158da846f') self.assertEqual(rows[0]['properties']['city'], u'Mcgregor') self.assertEqual(rows[0]['geometry']['coordinates'], [-97.3961768, 31.4432706]), self.assertEqual(rows[0]['properties']['street'], u'PULLEN ST') @@ -1081,7 +1081,7 @@ def test_single_ny_orange(self): self.assertEqual(len(rows), 8) self.assertEqual(rows[0]['properties']['id'], u'') self.assertEqual(rows[0]['properties']['number'], u'434') - self.assertEqual(rows[0]['properties']['hash'], u'a0127261b0619522') + self.assertEqual(rows[0]['properties']['hash'], u'41abd95de87dab64') self.assertEqual(rows[0]['properties']['city'], u'MONROE') self.assertEqual(rows[0]['geometry']['coordinates'], [-74.1926686, 41.3187728]) self.assertEqual(rows[0]['properties']['street'], u'') diff --git a/openaddr/tests/conform.py b/openaddr/tests/conform.py index 7b22955d..29d57ee9 100644 --- a/openaddr/tests/conform.py +++ b/openaddr/tests/conform.py @@ -437,6 +437,30 @@ def test_row_fxn_regexp(self): d = row_fxn_regexp(c, d, "street", c.data_source["conform"]["street"]) self.assertEqual(e, d) + def test_raw_properties(self): + d = SourceConfig(dict({ + "schema": 2, + "layers": { + "addresses": [{ + "name": "default", + "conform": { + "street": ["s1", "s2"], + "number": {"function": "regexp", "field": "full", "pattern": "^(\\d+)"}, + "id": "pid", + "lon": "x", + "lat": "y" + }, + "fingerprint": "0000" + }] + } + }), "addresses", "default") + + r = row_transform_and_convert(d, { + "s1": "MAPLE", "s2": "ST", "full": "123 MAPLE ST", "pid": "7", "x": "-119.2", "y": "39.3", + "UPRN": "34", "empty": "", "OA:extra": "skip", GEOM_FIELDNAME: "POINT (-119.2 39.3)"}) + self.assertEqual("123", r['properties']['number']) + self.assertEqual({"full": "123 MAPLE ST", "UPRN": "34"}, r['properties']['raw']) + def test_transform_and_convert(self): d = SourceConfig(dict({ "schema": 2, @@ -469,6 +493,7 @@ def test_transform_and_convert(self): 'hash': 'b3af08e447c7ed16', "id": "", "accuracy": "", + "raw": {}, }, 'geometry': { 'coordinates': [-119.2, 39.3], @@ -501,6 +526,7 @@ def test_transform_and_convert(self): "id": "", 'hash': 'd4681f7e1d34e6ed', "accuracy": "", + "raw": {}, }, 'geometry': { "type": "Point", @@ -550,6 +576,7 @@ def test_transform_and_convert(self): "building_name": "", 'hash': '591d7970b5753b0d', "accuracy": "", + "raw": {"s": "123 MAPLE ST"}, } }, r) @@ -590,6 +617,7 @@ def test_transform_and_convert(self): "building_name": "", 'hash': '7b1dc0b74cbc0162', "accuracy": "", + "raw": {}, } }, r)