Mongolia_Employment: Fix duplicate region formation and missing references in registered unemployment

statvar_imports/mongolia_imports/common_download_script.py

@@ -94,7 +94,25 @@
     },
     {
         "url": "https://data.1212.mn/api/v1/en/NSO/Labour%2C%20business/Labour/DT_NSO_0400_002V5.px",
-        "filename": "registered_unemployed_by_education_level_region_gender_month.csv"
+        "filename": "registered_unemployed_by_education_level_region_gender_month.csv",
+        # PxWeb API does not support negative/exclusion filters, so we explicitly list
+        # all 27 valid region codes to exclude legacy duplicate code "511" ("  Ulaanbaatar"),
+        # which collides with code "5" ("Ulaanbaatar") after whitespace stripping and has
+        # conflicting historical counts (2008-01 to 2011-09).
+        "query": [{
+            "code": "Бүс",
+            "selection": {
+                "filter": "item",
+                "values": [
+                    "0",
+                    "1", "181", "182", "183", "184", "185",
+                    "2", "261", "262", "263", "264", "265", "267",
+                    "3", "341", "342", "343", "344", "345", "346", "348",
+                    "4", "421", "422", "423",
+                    "5"
+                ]
+            }
+        }]
     }
 ]
 
@@ -159,14 +177,14 @@ def main(_):
     os.makedirs(demographics_dir, exist_ok=True)
     for table in DEMOGRAPHICS_TABLES:
         filepath = os.path.join(demographics_dir, table['filename'])
-        fetch_and_save_data(table['url'], filepath)
+        fetch_and_save_data(table['url'], filepath, table.get('query'))
 
     # Education Data
     education_dir = os.path.join(_SCRIPT_DIR, "mongolia_education", "input_files")
     os.makedirs(education_dir, exist_ok=True)
     for table in EDUCATION_TABLES:
         filepath = os.path.join(education_dir, table['filename'])
-        fetch_and_save_data(table['url'], filepath)
+        fetch_and_save_data(table['url'], filepath, table.get('query'))
 
     # Health Data
     health_dir = os.path.join(_SCRIPT_DIR, "mongolia_health", "input_files")
@@ -180,7 +198,7 @@ def main(_):
     os.makedirs(employment_dir, exist_ok=True)
     for table in EMPLOYMENT_TABLES:
         filepath = os.path.join(employment_dir, table['filename'])
-        fetch_and_save_data(table['url'], filepath)
+        fetch_and_save_data(table['url'], filepath, table.get('query'))
 
     logging.info("All tasks completed")
 

statvar_imports/mongolia_imports/common_download_script_test.py

@@ -155,6 +155,51 @@ def test_main_processes_all_25_tables(self, mock_makedirs, mock_fetch):
         common_download_script.main(None)
         self.assertEqual(mock_fetch.call_count, 25)
 
+    def test_registered_unemployed_query_excludes_duplicate_ulaanbaatar(self):
+        """Verifies registered unemployed table defines query excluding code 511."""
+        target_table = [
+            t
+            for t in common_download_script.EMPLOYMENT_TABLES
+            if t['filename']
+            == 'registered_unemployed_by_education_level_region_gender_month.csv'
+        ][0]
+        self.assertIn('query', target_table)
+        query = target_table['query']
+        self.assertEqual(query[0]['code'], 'Бүс')
+        values = query[0]['selection']['values']
+        self.assertIn('5', values)
+        self.assertNotIn('511', values)
+        expected_values = [
+            "0",
+            "1", "181", "182", "183", "184", "185",
+            "2", "261", "262", "263", "264", "265", "267",
+            "3", "341", "342", "343", "344", "345", "346", "348",
+            "4", "421", "422", "423",
+            "5"
+        ]
+        self.assertEqual(values, expected_values)
+
+    @mock.patch('common_download_script.fetch_and_save_data')
+    @mock.patch('os.makedirs')
+    def test_main_passes_query_for_registered_unemployed(
+        self, mock_makedirs, mock_fetch
+    ):
+        """Verifies that main() forwards query filter for registered unemployed."""
+        common_download_script.main(None)
+        unemployed_calls = [
+            c
+            for c in mock_fetch.call_args_list
+            if 'registered_unemployed_by_education_level_region_gender_month.csv'
+            in c[0][1]
+        ]
+        self.assertEqual(len(unemployed_calls), 1)
+        args, kwargs = unemployed_calls[0]
+        query = args[2] if len(args) > 2 else kwargs.get('query')
+        self.assertIsNotNone(query)
+        self.assertEqual(query[0]['code'], 'Бүс')
+        self.assertNotIn('511', query[0]['selection']['values'])
+
 
 if __name__ == '__main__':
     unittest.main()
+