From ea4cdd2362df7ed842426a56eeaa993c07b73a28 Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Tue, 7 Apr 2026 07:21:33 +0000 Subject: [PATCH 1/9] Fixed Code for 3 files --- scripts/un/energy/download.py | 4 ++-- scripts/un/energy/process.py | 3 +++ scripts/un/energy/process_test.py | 1 - 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/un/energy/download.py b/scripts/un/energy/download.py index 4542b4068c..b3dab19d4c 100644 --- a/scripts/un/energy/download.py +++ b/scripts/un/energy/download.py @@ -90,7 +90,6 @@ def download_energy_dataset( str(supported_datasets)) return output_files # Download data in batches of years as the download has a limit of 100k rows. - years_list = list(range(start_year, years_per_batch + 1)) years_list = [str(y) for y in range(start_year, end_year + 1)] batch_years = [ years_list[i:i + years_per_batch] @@ -117,7 +116,8 @@ def download_energy_dataset( if download_successful: logging.info(f"Download of '{download_url}' completed.") for f in os.listdir(output): - output_files.append(os.path.join(output, f)) + if f.endswith('.csv'): + output_files.append(os.path.join(output, f)) else: logging.fatal(f"Download or processing of '{download_url}' failed") return output_files diff --git a/scripts/un/energy/process.py b/scripts/un/energy/process.py index eedbe872d8..5791f7644d 100644 --- a/scripts/un/energy/process.py +++ b/scripts/un/energy/process.py @@ -545,6 +545,9 @@ def process(in_paths: list, with open(mcf_file_path, 'w+', newline='') as f_out_mcf: # Process each CSV input file, one row at a time. for in_file in in_paths: + if not in_file.endswith('.csv'): + logging.info(f'Skipping non-CSV file: {in_file}') + continue logging.info(f'Processing data file: {in_file}') with open(in_file) as csvfile: counters['input_files'] += 1 diff --git a/scripts/un/energy/process_test.py b/scripts/un/energy/process_test.py index 776e45ccbf..bb7fb81c77 100644 --- a/scripts/un/energy/process_test.py +++ b/scripts/un/energy/process_test.py @@ -67,5 +67,4 @@ def test_un_energy_process(self): if __name__ == '__main__': - app.run() unittest.main() From 91d22a72b80b57ddae12b3a9d23be9f72ec9430d Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Wed, 8 Apr 2026 10:59:08 +0000 Subject: [PATCH 2/9] NameError Resolved --- scripts/un/energy/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/un/energy/download.py b/scripts/un/energy/download.py index b3dab19d4c..1ef6e37f13 100644 --- a/scripts/un/energy/download.py +++ b/scripts/un/energy/download.py @@ -83,6 +83,7 @@ def download_energy_dataset( Returns: A list of output files downloaded. """ + output_files = [] supported_datasets = get_all_energy_source_codes() if energy_dataset not in supported_datasets: logging.info( @@ -95,7 +96,6 @@ def download_energy_dataset( years_list[i:i + years_per_batch] for i in range(0, len(years_list), years_per_batch) ] - output_files = [] for year_batch in batch_years: start_year = year_batch[0] end_year = year_batch[-1] From 78845d95c50fbaaba37b2f66b9bb6ebe3df225c5 Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Wed, 9 Sep 2026 10:14:20 +0000 Subject: [PATCH 3/9] Fix Mongolia_Employment duplicate Ulaanbaatar and missing references - Filter out erroneous duplicate indented Region code 511 (' Ulaanbaatar') in common_download_script.py for registered_unemployed_by_education_level_region_gender_month.csv - Pass table.get('query') in common_download_script.py for EMPLOYMENT_TABLES - Add defense-in-depth sanitization in run.sh to strip duplicate indented Ulaanbaatar rows from input file if present - Add unit test in common_download_script_test.py verifying query configuration --- .../mongolia_imports/common_download_script.py | 16 ++++++++++++++-- .../common_download_script_test.py | 16 ++++++++++++++++ .../mongolia_imports/mongolia_employment/run.sh | 7 +++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/statvar_imports/mongolia_imports/common_download_script.py b/statvar_imports/mongolia_imports/common_download_script.py index 828e3d2dff..388e9cbe09 100644 --- a/statvar_imports/mongolia_imports/common_download_script.py +++ b/statvar_imports/mongolia_imports/common_download_script.py @@ -94,7 +94,19 @@ }, { "url": "https://data.1212.mn/api/v1/en/NSO/Labour%2C%20business/Labour/DT_NSO_0400_002V5.px", - "filename": "registered_unemployed_by_education_level_region_gender_month.csv" + "filename": "registered_unemployed_by_education_level_region_gender_month.csv", + "query": [{ + "code": "Бүс", + "selection": { + "filter": "item", + "values": [ + "0", "1", "183", "182", "181", "185", "184", "2", + "265", "264", "263", "261", "262", "267", "3", "342", + "345", "344", "348", "346", "343", "341", "4", "421", + "422", "423", "5" + ] + } + }] } ] @@ -180,7 +192,7 @@ def main(_): os.makedirs(employment_dir, exist_ok=True) for table in EMPLOYMENT_TABLES: filepath = os.path.join(employment_dir, table['filename']) - fetch_and_save_data(table['url'], filepath) + fetch_and_save_data(table['url'], filepath, table.get('query')) logging.info("All tasks completed") diff --git a/statvar_imports/mongolia_imports/common_download_script_test.py b/statvar_imports/mongolia_imports/common_download_script_test.py index 087679131e..63e3b1d337 100644 --- a/statvar_imports/mongolia_imports/common_download_script_test.py +++ b/statvar_imports/mongolia_imports/common_download_script_test.py @@ -155,6 +155,22 @@ def test_main_processes_all_25_tables(self, mock_makedirs, mock_fetch): common_download_script.main(None) self.assertEqual(mock_fetch.call_count, 25) + def test_registered_unemployed_query_excludes_duplicate_ulaanbaatar(self): + """Verifies registered unemployed table defines query excluding code 511.""" + target_table = [ + t + for t in common_download_script.EMPLOYMENT_TABLES + if t['filename'] + == 'registered_unemployed_by_education_level_region_gender_month.csv' + ][0] + self.assertIn('query', target_table) + query = target_table['query'] + self.assertEqual(query[0]['code'], 'Бүс') + values = query[0]['selection']['values'] + self.assertIn('5', values) + self.assertNotIn('511', values) + if __name__ == '__main__': unittest.main() + diff --git a/statvar_imports/mongolia_imports/mongolia_employment/run.sh b/statvar_imports/mongolia_imports/mongolia_employment/run.sh index 94e2ffb61a..3a867c8dce 100644 --- a/statvar_imports/mongolia_imports/mongolia_employment/run.sh +++ b/statvar_imports/mongolia_imports/mongolia_employment/run.sh @@ -14,7 +14,14 @@ python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --inp python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/labour_underutilization_by_region_gender_and_agegroup.csv --pv_map=$SCRIPT_PATH/labour_underutilization_by_region_gender_and_agegroup_pvmap.csv --config_file=$SCRIPT_PATH/metadata.csv --output_path=$SCRIPT_PATH/output_files/labour_underutilization_by_region_gender_and_agegroup_output --output_counters=$SCRIPT_PATH/counters/labour_underutilization_by_region_gender_and_agegroup_output_counters.csv --places_resolved_csv=$SCRIPT_PATH/places_resolved.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf || { echo "Error: Processing labour_under_utilization_by_region_gender_and_agegroup failed!"; exit 1; } +INPUT_5="$SCRIPT_PATH/input_files/registered_unemployed_by_education_level_region_gender_month.csv" +if [ -f "$INPUT_5" ] && grep -q '^"[^"]*"," Ulaanbaatar"' "$INPUT_5"; then + echo "Stripping duplicate indented Ulaanbaatar entries from $INPUT_5" + sed -i '/^"[^"]*"," Ulaanbaatar"/d' "$INPUT_5" +fi + python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/registered_unemployed_by_education_level_region_gender_month.csv --pv_map=$SCRIPT_PATH/registered_unemployed_by_education_level_region_gender_month_pvmap.csv --config_file=$SCRIPT_PATH/metadata.csv --output_path=$SCRIPT_PATH/output_files/registered_unemployed_by_education_level_region_gender_month_output --output_counters=$SCRIPT_PATH/counters/registered_unemployed_by_education_level_region_gender_month_output_counters.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=$SCRIPT_PATH/places_resolved.csv || { echo "Error: Processing registered_unemployed_by_education_level_region_gender_month failed!"; exit 1; } + echo "All processing steps completed successfully." exit 0 From fd1a4e18618bac2bd066b81f7cd91ef08d25025e Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Thu, 10 Sep 2026 07:58:27 +0000 Subject: [PATCH 4/9] Use temporary file for in-place edit in run.sh to support macOS BSD sed --- statvar_imports/mongolia_imports/mongolia_employment/run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/statvar_imports/mongolia_imports/mongolia_employment/run.sh b/statvar_imports/mongolia_imports/mongolia_employment/run.sh index 3a867c8dce..59cdf3edfa 100644 --- a/statvar_imports/mongolia_imports/mongolia_employment/run.sh +++ b/statvar_imports/mongolia_imports/mongolia_employment/run.sh @@ -17,7 +17,8 @@ python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --inp INPUT_5="$SCRIPT_PATH/input_files/registered_unemployed_by_education_level_region_gender_month.csv" if [ -f "$INPUT_5" ] && grep -q '^"[^"]*"," Ulaanbaatar"' "$INPUT_5"; then echo "Stripping duplicate indented Ulaanbaatar entries from $INPUT_5" - sed -i '/^"[^"]*"," Ulaanbaatar"/d' "$INPUT_5" + TMP_FILE="$(mktemp)" + sed '/^"[^"]*"," Ulaanbaatar"/d' "$INPUT_5" > "$TMP_FILE" && mv "$TMP_FILE" "$INPUT_5" fi python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/registered_unemployed_by_education_level_region_gender_month.csv --pv_map=$SCRIPT_PATH/registered_unemployed_by_education_level_region_gender_month_pvmap.csv --config_file=$SCRIPT_PATH/metadata.csv --output_path=$SCRIPT_PATH/output_files/registered_unemployed_by_education_level_region_gender_month_output --output_counters=$SCRIPT_PATH/counters/registered_unemployed_by_education_level_region_gender_month_output_counters.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=$SCRIPT_PATH/places_resolved.csv || { echo "Error: Processing registered_unemployed_by_education_level_region_gender_month failed!"; exit 1; } From 083e0777eb204c684f0152e1780dab8b8b5180cd Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Thu, 10 Sep 2026 09:06:32 +0000 Subject: [PATCH 5/9] Mongolia_Employment: Address code review findings for PR #2216 - Use co-located temporary file, atomic move, non-empty check, and exit trap in run.sh - Forward table.get('query') uniformly across all domain table loops in common_download_script.py - Add unit test verifying runtime query parameter propagation in common_download_script_test.py - Add MAX_DATE_CONSISTENT and SQL_VALIDATOR date freshness rules to validation_config.json --- .../common_download_script.py | 4 ++-- .../common_download_script_test.py | 20 +++++++++++++++++++ .../mongolia_employment/run.sh | 8 ++++++-- .../validation_config.json | 15 ++++++++++++++ 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/statvar_imports/mongolia_imports/common_download_script.py b/statvar_imports/mongolia_imports/common_download_script.py index 388e9cbe09..1083076786 100644 --- a/statvar_imports/mongolia_imports/common_download_script.py +++ b/statvar_imports/mongolia_imports/common_download_script.py @@ -171,14 +171,14 @@ def main(_): os.makedirs(demographics_dir, exist_ok=True) for table in DEMOGRAPHICS_TABLES: filepath = os.path.join(demographics_dir, table['filename']) - fetch_and_save_data(table['url'], filepath) + fetch_and_save_data(table['url'], filepath, table.get('query')) # Education Data education_dir = os.path.join(_SCRIPT_DIR, "mongolia_education", "input_files") os.makedirs(education_dir, exist_ok=True) for table in EDUCATION_TABLES: filepath = os.path.join(education_dir, table['filename']) - fetch_and_save_data(table['url'], filepath) + fetch_and_save_data(table['url'], filepath, table.get('query')) # Health Data health_dir = os.path.join(_SCRIPT_DIR, "mongolia_health", "input_files") diff --git a/statvar_imports/mongolia_imports/common_download_script_test.py b/statvar_imports/mongolia_imports/common_download_script_test.py index 63e3b1d337..8cf852ef99 100644 --- a/statvar_imports/mongolia_imports/common_download_script_test.py +++ b/statvar_imports/mongolia_imports/common_download_script_test.py @@ -170,6 +170,26 @@ def test_registered_unemployed_query_excludes_duplicate_ulaanbaatar(self): self.assertIn('5', values) self.assertNotIn('511', values) + @mock.patch('common_download_script.fetch_and_save_data') + @mock.patch('os.makedirs') + def test_main_passes_query_for_registered_unemployed( + self, mock_makedirs, mock_fetch + ): + """Verifies that main() forwards query filter for registered unemployed.""" + common_download_script.main(None) + unemployed_calls = [ + c + for c in mock_fetch.call_args_list + if 'registered_unemployed_by_education_level_region_gender_month.csv' + in c[0][1] + ] + self.assertEqual(len(unemployed_calls), 1) + args, kwargs = unemployed_calls[0] + query = args[2] if len(args) > 2 else kwargs.get('query') + self.assertIsNotNone(query) + self.assertEqual(query[0]['code'], 'Бүс') + self.assertNotIn('511', query[0]['selection']['values']) + if __name__ == '__main__': unittest.main() diff --git a/statvar_imports/mongolia_imports/mongolia_employment/run.sh b/statvar_imports/mongolia_imports/mongolia_employment/run.sh index 59cdf3edfa..61e89e4f0c 100644 --- a/statvar_imports/mongolia_imports/mongolia_employment/run.sh +++ b/statvar_imports/mongolia_imports/mongolia_employment/run.sh @@ -17,8 +17,12 @@ python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --inp INPUT_5="$SCRIPT_PATH/input_files/registered_unemployed_by_education_level_region_gender_month.csv" if [ -f "$INPUT_5" ] && grep -q '^"[^"]*"," Ulaanbaatar"' "$INPUT_5"; then echo "Stripping duplicate indented Ulaanbaatar entries from $INPUT_5" - TMP_FILE="$(mktemp)" - sed '/^"[^"]*"," Ulaanbaatar"/d' "$INPUT_5" > "$TMP_FILE" && mv "$TMP_FILE" "$INPUT_5" + TMP_FILE="$INPUT_5.tmp.$$" + trap 'rm -f "$TMP_FILE"' EXIT + if sed '/^"[^"]*"," Ulaanbaatar"/d' "$INPUT_5" > "$TMP_FILE" && [ -s "$TMP_FILE" ]; then + mv "$TMP_FILE" "$INPUT_5" + fi + trap - EXIT fi python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/registered_unemployed_by_education_level_region_gender_month.csv --pv_map=$SCRIPT_PATH/registered_unemployed_by_education_level_region_gender_month_pvmap.csv --config_file=$SCRIPT_PATH/metadata.csv --output_path=$SCRIPT_PATH/output_files/registered_unemployed_by_education_level_region_gender_month_output --output_counters=$SCRIPT_PATH/counters/registered_unemployed_by_education_level_region_gender_month_output_counters.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=$SCRIPT_PATH/places_resolved.csv || { echo "Error: Processing registered_unemployed_by_education_level_region_gender_month failed!"; exit 1; } diff --git a/statvar_imports/mongolia_imports/mongolia_employment/validation_config.json b/statvar_imports/mongolia_imports/mongolia_employment/validation_config.json index 87e2625c42..86bb19c42a 100644 --- a/statvar_imports/mongolia_imports/mongolia_employment/validation_config.json +++ b/statvar_imports/mongolia_imports/mongolia_employment/validation_config.json @@ -116,6 +116,21 @@ "golden_files": "../../../../golden_data/input5_golden_observations.csv", "input_files": "../../../../output_files/registered_unemployed_by_education_level_region_gender_month_output.csv" } + }, + { + "rule_id": "check_max_date_consistent", + "description": "Checks that the latest date is the same for all StatVars.", + "validator": "MAX_DATE_CONSISTENT", + "params": {} + }, + { + "rule_id": "check_max_date_freshness", + "description": "Checks that the latest date does not regress below 2024.", + "validator": "SQL_VALIDATOR", + "params": { + "query": "SELECT StatVar, MaxDate FROM stats", + "condition": "MaxDate >= '2024'" + } } ] } \ No newline at end of file From 63b90bab93d875d29bbdedf8037d4d6f266ca008 Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Thu, 10 Sep 2026 18:37:52 +0000 Subject: [PATCH 6/9] Mongolia_Employment: Remove check_max_date_consistent and check_max_date_freshness from validation_config.json --- .../mongolia_employment/validation_config.json | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/statvar_imports/mongolia_imports/mongolia_employment/validation_config.json b/statvar_imports/mongolia_imports/mongolia_employment/validation_config.json index 86bb19c42a..6c2e7b0fc9 100644 --- a/statvar_imports/mongolia_imports/mongolia_employment/validation_config.json +++ b/statvar_imports/mongolia_imports/mongolia_employment/validation_config.json @@ -116,21 +116,6 @@ "golden_files": "../../../../golden_data/input5_golden_observations.csv", "input_files": "../../../../output_files/registered_unemployed_by_education_level_region_gender_month_output.csv" } - }, - { - "rule_id": "check_max_date_consistent", - "description": "Checks that the latest date is the same for all StatVars.", - "validator": "MAX_DATE_CONSISTENT", - "params": {} - }, - { - "rule_id": "check_max_date_freshness", - "description": "Checks that the latest date does not regress below 2024.", - "validator": "SQL_VALIDATOR", - "params": { - "query": "SELECT StatVar, MaxDate FROM stats", - "condition": "MaxDate >= '2024'" - } } ] -} \ No newline at end of file +} From d398ef32c0576bc94a18172af9c1cc6d1ad807c6 Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Fri, 11 Sep 2026 04:11:04 +0000 Subject: [PATCH 7/9] Mongolia_Employment: Restore validation_config.json to match origin/master --- .../mongolia_imports/mongolia_employment/validation_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statvar_imports/mongolia_imports/mongolia_employment/validation_config.json b/statvar_imports/mongolia_imports/mongolia_employment/validation_config.json index 6c2e7b0fc9..87e2625c42 100644 --- a/statvar_imports/mongolia_imports/mongolia_employment/validation_config.json +++ b/statvar_imports/mongolia_imports/mongolia_employment/validation_config.json @@ -118,4 +118,4 @@ } } ] -} +} \ No newline at end of file From fafb0e535e7cfd75731681df16aa743324fcea7f Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Fri, 11 Sep 2026 10:41:02 +0000 Subject: [PATCH 8/9] Mongolia_Employment: Restore run.sh to match origin/master --- .../mongolia_imports/mongolia_employment/run.sh | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/statvar_imports/mongolia_imports/mongolia_employment/run.sh b/statvar_imports/mongolia_imports/mongolia_employment/run.sh index 61e89e4f0c..94e2ffb61a 100644 --- a/statvar_imports/mongolia_imports/mongolia_employment/run.sh +++ b/statvar_imports/mongolia_imports/mongolia_employment/run.sh @@ -14,19 +14,7 @@ python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --inp python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/labour_underutilization_by_region_gender_and_agegroup.csv --pv_map=$SCRIPT_PATH/labour_underutilization_by_region_gender_and_agegroup_pvmap.csv --config_file=$SCRIPT_PATH/metadata.csv --output_path=$SCRIPT_PATH/output_files/labour_underutilization_by_region_gender_and_agegroup_output --output_counters=$SCRIPT_PATH/counters/labour_underutilization_by_region_gender_and_agegroup_output_counters.csv --places_resolved_csv=$SCRIPT_PATH/places_resolved.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf || { echo "Error: Processing labour_under_utilization_by_region_gender_and_agegroup failed!"; exit 1; } -INPUT_5="$SCRIPT_PATH/input_files/registered_unemployed_by_education_level_region_gender_month.csv" -if [ -f "$INPUT_5" ] && grep -q '^"[^"]*"," Ulaanbaatar"' "$INPUT_5"; then - echo "Stripping duplicate indented Ulaanbaatar entries from $INPUT_5" - TMP_FILE="$INPUT_5.tmp.$$" - trap 'rm -f "$TMP_FILE"' EXIT - if sed '/^"[^"]*"," Ulaanbaatar"/d' "$INPUT_5" > "$TMP_FILE" && [ -s "$TMP_FILE" ]; then - mv "$TMP_FILE" "$INPUT_5" - fi - trap - EXIT -fi - python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/registered_unemployed_by_education_level_region_gender_month.csv --pv_map=$SCRIPT_PATH/registered_unemployed_by_education_level_region_gender_month_pvmap.csv --config_file=$SCRIPT_PATH/metadata.csv --output_path=$SCRIPT_PATH/output_files/registered_unemployed_by_education_level_region_gender_month_output --output_counters=$SCRIPT_PATH/counters/registered_unemployed_by_education_level_region_gender_month_output_counters.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=$SCRIPT_PATH/places_resolved.csv || { echo "Error: Processing registered_unemployed_by_education_level_region_gender_month failed!"; exit 1; } - echo "All processing steps completed successfully." exit 0 From e400e6d0d5efdd8174c870876ada9b5ac494c258 Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Thu, 17 Sep 2026 12:59:14 +0000 Subject: [PATCH 9/9] Mongolia_Employment: Sort region codes and document exclusion of code 511 in common_download_script --- .../mongolia_imports/common_download_script.py | 14 ++++++++++---- .../common_download_script_test.py | 9 +++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/statvar_imports/mongolia_imports/common_download_script.py b/statvar_imports/mongolia_imports/common_download_script.py index 1083076786..7c4e9ff233 100644 --- a/statvar_imports/mongolia_imports/common_download_script.py +++ b/statvar_imports/mongolia_imports/common_download_script.py @@ -95,15 +95,21 @@ { "url": "https://data.1212.mn/api/v1/en/NSO/Labour%2C%20business/Labour/DT_NSO_0400_002V5.px", "filename": "registered_unemployed_by_education_level_region_gender_month.csv", + # PxWeb API does not support negative/exclusion filters, so we explicitly list + # all 27 valid region codes to exclude legacy duplicate code "511" (" Ulaanbaatar"), + # which collides with code "5" ("Ulaanbaatar") after whitespace stripping and has + # conflicting historical counts (2008-01 to 2011-09). "query": [{ "code": "Бүс", "selection": { "filter": "item", "values": [ - "0", "1", "183", "182", "181", "185", "184", "2", - "265", "264", "263", "261", "262", "267", "3", "342", - "345", "344", "348", "346", "343", "341", "4", "421", - "422", "423", "5" + "0", + "1", "181", "182", "183", "184", "185", + "2", "261", "262", "263", "264", "265", "267", + "3", "341", "342", "343", "344", "345", "346", "348", + "4", "421", "422", "423", + "5" ] } }] diff --git a/statvar_imports/mongolia_imports/common_download_script_test.py b/statvar_imports/mongolia_imports/common_download_script_test.py index 8cf852ef99..dc80f3d85a 100644 --- a/statvar_imports/mongolia_imports/common_download_script_test.py +++ b/statvar_imports/mongolia_imports/common_download_script_test.py @@ -169,6 +169,15 @@ def test_registered_unemployed_query_excludes_duplicate_ulaanbaatar(self): values = query[0]['selection']['values'] self.assertIn('5', values) self.assertNotIn('511', values) + expected_values = [ + "0", + "1", "181", "182", "183", "184", "185", + "2", "261", "262", "263", "264", "265", "267", + "3", "341", "342", "343", "344", "345", "346", "348", + "4", "421", "422", "423", + "5" + ] + self.assertEqual(values, expected_values) @mock.patch('common_download_script.fetch_and_save_data') @mock.patch('os.makedirs')