diff --git a/statvar_imports/mongolia_imports/common_download_script.py b/statvar_imports/mongolia_imports/common_download_script.py index 828e3d2dff..7c4e9ff233 100644 --- a/statvar_imports/mongolia_imports/common_download_script.py +++ b/statvar_imports/mongolia_imports/common_download_script.py @@ -94,7 +94,25 @@ }, { "url": "https://data.1212.mn/api/v1/en/NSO/Labour%2C%20business/Labour/DT_NSO_0400_002V5.px", - "filename": "registered_unemployed_by_education_level_region_gender_month.csv" + "filename": "registered_unemployed_by_education_level_region_gender_month.csv", + # PxWeb API does not support negative/exclusion filters, so we explicitly list + # all 27 valid region codes to exclude legacy duplicate code "511" (" Ulaanbaatar"), + # which collides with code "5" ("Ulaanbaatar") after whitespace stripping and has + # conflicting historical counts (2008-01 to 2011-09). + "query": [{ + "code": "Бүс", + "selection": { + "filter": "item", + "values": [ + "0", + "1", "181", "182", "183", "184", "185", + "2", "261", "262", "263", "264", "265", "267", + "3", "341", "342", "343", "344", "345", "346", "348", + "4", "421", "422", "423", + "5" + ] + } + }] } ] @@ -159,14 +177,14 @@ def main(_): os.makedirs(demographics_dir, exist_ok=True) for table in DEMOGRAPHICS_TABLES: filepath = os.path.join(demographics_dir, table['filename']) - fetch_and_save_data(table['url'], filepath) + fetch_and_save_data(table['url'], filepath, table.get('query')) # Education Data education_dir = os.path.join(_SCRIPT_DIR, "mongolia_education", "input_files") os.makedirs(education_dir, exist_ok=True) for table in EDUCATION_TABLES: filepath = os.path.join(education_dir, table['filename']) - fetch_and_save_data(table['url'], filepath) + fetch_and_save_data(table['url'], filepath, table.get('query')) # Health Data health_dir = os.path.join(_SCRIPT_DIR, "mongolia_health", "input_files") @@ -180,7 +198,7 @@ def main(_): os.makedirs(employment_dir, exist_ok=True) for table in EMPLOYMENT_TABLES: filepath = os.path.join(employment_dir, table['filename']) - fetch_and_save_data(table['url'], filepath) + fetch_and_save_data(table['url'], filepath, table.get('query')) logging.info("All tasks completed") diff --git a/statvar_imports/mongolia_imports/common_download_script_test.py b/statvar_imports/mongolia_imports/common_download_script_test.py index 087679131e..dc80f3d85a 100644 --- a/statvar_imports/mongolia_imports/common_download_script_test.py +++ b/statvar_imports/mongolia_imports/common_download_script_test.py @@ -155,6 +155,51 @@ def test_main_processes_all_25_tables(self, mock_makedirs, mock_fetch): common_download_script.main(None) self.assertEqual(mock_fetch.call_count, 25) + def test_registered_unemployed_query_excludes_duplicate_ulaanbaatar(self): + """Verifies registered unemployed table defines query excluding code 511.""" + target_table = [ + t + for t in common_download_script.EMPLOYMENT_TABLES + if t['filename'] + == 'registered_unemployed_by_education_level_region_gender_month.csv' + ][0] + self.assertIn('query', target_table) + query = target_table['query'] + self.assertEqual(query[0]['code'], 'Бүс') + values = query[0]['selection']['values'] + self.assertIn('5', values) + self.assertNotIn('511', values) + expected_values = [ + "0", + "1", "181", "182", "183", "184", "185", + "2", "261", "262", "263", "264", "265", "267", + "3", "341", "342", "343", "344", "345", "346", "348", + "4", "421", "422", "423", + "5" + ] + self.assertEqual(values, expected_values) + + @mock.patch('common_download_script.fetch_and_save_data') + @mock.patch('os.makedirs') + def test_main_passes_query_for_registered_unemployed( + self, mock_makedirs, mock_fetch + ): + """Verifies that main() forwards query filter for registered unemployed.""" + common_download_script.main(None) + unemployed_calls = [ + c + for c in mock_fetch.call_args_list + if 'registered_unemployed_by_education_level_region_gender_month.csv' + in c[0][1] + ] + self.assertEqual(len(unemployed_calls), 1) + args, kwargs = unemployed_calls[0] + query = args[2] if len(args) > 2 else kwargs.get('query') + self.assertIsNotNone(query) + self.assertEqual(query[0]['code'], 'Бүс') + self.assertNotIn('511', query[0]['selection']['values']) + if __name__ == '__main__': unittest.main() +