From 5743b57fd6913b40f815252d254794f8fe489a1f Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Fri, 4 Sep 2026 13:27:32 +0000 Subject: [PATCH 1/9] Modified download script and handle deletions --- .../environment_pvmap.csv | 2 + .../india_rbi_state_statistics/manifest.json | 57 ++++++++++----- .../rbi_download.py | 50 +++++++------ .../rbi_download_test.py | 71 +++++++++++++++++++ .../rbi_places_resolver.csv | 3 +- .../validation_config.json | 13 ++++ 6 files changed, 155 insertions(+), 41 deletions(-) create mode 100644 statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py create mode 100644 statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv index 0a1f8a72a5..4ea5beff8a 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv @@ -6,6 +6,8 @@ Table 105: STATE-WISE EXPENDITURE ON RELIEF ON NATURAL CALAMITIES,measuredProper Table 106: STATE-WISE SUSTAINABLE DEVELOPMENT GOALS (SDGs) SCORE,measuredProperty,score,populationType,SustainableDevelopmentGoals,name,"""Sustainable Development Goals Score""",,,,,,,, State/ Union Territory,observationAbout,{Data},,,,,,,,,,,, State/Union Territory,observationAbout,{Data},,,,,,,,,,,, +"Haryana, Delhi & Chandigarh",observationAbout,dcid:wikidataId/Q1174 +"Haryana, Delhi and Chandigarh",observationAbout,dcid:wikidataId/Q1174 1987,observationDate,1987,value,{Number},,,,,,,,,, 1989,observationDate,1989,value,{Number},,,,,,,,,, 1991,observationDate,1991,value,{Number},,,,,,,,,, diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json index efeca24ba2..0542cc79e7 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json @@ -20,78 +20,97 @@ "import_inputs": [ { "template_mcf": "output_files/agriculture/agriculture_output.tmcf", - "cleaned_csv": "output_files/agriculture/agriculture_output.csv" + "cleaned_csv": "output_files/agriculture/agriculture_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.csv" + "cleaned_csv": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_forest_cover_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_forest_cover_output.csv" + "cleaned_csv": "output_files/environment/state_wise_forest_cover_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_sdg_score_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_sdg_score_output.csv" + "cleaned_csv": "output_files/environment/state_wise_sdg_score_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_tree_cover_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_tree_cover_output.csv" + "cleaned_csv": "output_files/environment/state_wise_tree_cover_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/environment/sub_division_wise_annual_rainfall_output.tmcf", - "cleaned_csv": "output_files/environment/sub_division_wise_annual_rainfall_output.csv" + "cleaned_csv": "output_files/environment/sub_division_wise_annual_rainfall_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/price_and_wages/price_and_wages_output.tmcf", - "cleaned_csv": "output_files/price_and_wages/price_and_wages_output.csv" + "cleaned_csv": "output_files/price_and_wages/price_and_wages_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_availability_of_power_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_availability_of_power_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_availability_of_power_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_installed_capacity_of_power_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_installed_capacity_of_power_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_installed_capacity_of_power_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_length_of_national_highways_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_length_of_national_highways_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_length_of_national_highways_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_length_of_roads_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_length_of_roads_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_length_of_roads_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_length_of_state_highways_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_length_of_state_highways_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_length_of_state_highways_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_per_capita_availability_of_power_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_per_capita_availability_of_power_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_per_capita_availability_of_power_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_power_requirement_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_power_requirement_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_power_requirement_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_railway_route_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_railway_route_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_railway_route_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output.csv", + "node_mcf": "output_files/*/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_telephones_per_100_population_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_telephones_per_100_population_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_telephones_per_100_population_output.csv", + "node_mcf": "output_files/*/*.mcf" } ], - "cron_schedule": "0 10 * * 1" + "cron_schedule": "0 10 * * 1", + "validation_config_file": "validation_config.json" } ] } \ No newline at end of file diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py index e30edc2db1..b6c7c654aa 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import tempfile from absl import app, logging, flags import sys from google.cloud import storage @@ -80,26 +81,33 @@ def preprocess_files(directory_path): logging.info(f"Processing file: {file_path}") try: all_sheets_data = pd.read_excel(file_path, sheet_name=None, header=None) - with pd.ExcelWriter(file_path, engine='openpyxl') as writer: - for sheet_name, df in all_sheets_data.items(): - df = df.map(lambda x: str(x).replace('*', '').replace('@', '').strip()) - mask = df.eq('State/Union Territory').any(axis=1) - state_positions = df[mask] == 'State/Union Territory' - - def safe_to_numeric(val): - val_str = str(val) - if val_str.isdigit(): - return int(val_str) - try: - float_val = float(val_str) - return float_val - except ValueError: - return val - - df_num = df[mask].applymap(safe_to_numeric) - converted = df_num.mask(state_positions, 'State/Union Territory') - df.loc[mask, :] = converted - df.to_excel(writer, sheet_name=sheet_name, index=False, header=False) + def safe_to_numeric(val): + val_str = str(val) + if val_str.isdigit(): + return int(val_str) + try: + float_val = float(val_str) + return float_val + except ValueError: + return val + + for sheet_name, df in all_sheets_data.items(): + df = df.map(lambda x: str(x).replace('*', '').replace('@', '').strip()) + mask = df.eq('State/Union Territory').any(axis=1) + state_positions = df[mask] == 'State/Union Territory' + + df_num = df[mask].map(safe_to_numeric) + converted = df_num.mask(state_positions, 'State/Union Territory') + df = df.astype(object) + df.loc[mask, :] = converted + all_sheets_data[sheet_name] = df + + with tempfile.TemporaryDirectory(dir=directory_path) as output_dir: + output_path = os.path.join(output_dir, file_name) + with pd.ExcelWriter(output_path, engine='openpyxl') as writer: + for sheet_name, df in all_sheets_data.items(): + df.to_excel(writer, sheet_name=sheet_name, index=False, header=False) + os.replace(output_path, file_path) except Exception as e: logging.fatal(f"Error processing {file_name}: {e}") @@ -119,4 +127,4 @@ def main(_): if __name__ == "__main__": - app.run(main) + app.run(main) \ No newline at end of file diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py new file mode 100644 index 0000000000..d7894865cd --- /dev/null +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py @@ -0,0 +1,71 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pathlib +import tempfile +from unittest import mock + +from absl.testing import absltest +import pandas as pd + +from statvar_imports.database_on_indian_economy.india_rbi_state_statistics import rbi_download + + +class PreprocessFilesTest(absltest.TestCase): + + def test_preserves_workbook_and_original_error(self): + with tempfile.TemporaryDirectory() as directory: + workbook = pathlib.Path(directory) / 'source.xlsx' + workbook.write_bytes(b'original workbook') + sheets = {'Sheet1': pd.DataFrame([['State/Union Territory']])} + + with mock.patch.object( + rbi_download.pd, 'read_excel', return_value=sheets), \ + mock.patch.object( + pd.DataFrame, + 'map', + side_effect=ValueError('transform failed')), \ + mock.patch.object(rbi_download.logging, 'fatal') as fatal: + rbi_download.preprocess_files(directory) + + self.assertEqual(workbook.read_bytes(), b'original workbook') + fatal.assert_called_once_with( + 'Error processing source.xlsx: transform failed') + + def test_preprocess_files_converts_numeric_headers(self): + with tempfile.TemporaryDirectory() as directory: + file_path = pathlib.Path(directory) / 'source.xlsx' + initial_df = pd.DataFrame( + [['State/Union Territory', '2015*', '2016@', '2017-18'], + ['Andhra Pradesh', '10.5', '20.0', '30.5']]) + with pd.ExcelWriter(file_path, engine='openpyxl') as writer: + initial_df.to_excel(writer, + sheet_name='Sheet1', + index=False, + header=False) + + rbi_download.preprocess_files(directory) + + processed = pd.read_excel(file_path, + sheet_name='Sheet1', + header=None) + self.assertEqual(processed.iloc[0, 0], 'State/Union Territory') + self.assertEqual(processed.iloc[0, 1], 2015) + self.assertEqual(processed.iloc[0, 2], 2016) + self.assertEqual(processed.iloc[0, 3], '2017-18') + self.assertEqual(processed.iloc[1, 0], 'Andhra Pradesh') + + +if __name__ == '__main__': + absltest.main() diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv index a67a4fa4f1..2b8594585f 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv @@ -8,7 +8,7 @@ Chandigarh,wikidataId/Q43433 Chhattisgarh,wikidataId/Q1168 Dadra & Nagar Haveli,wikidataId/Q46107 Daman & Diu,wikidataId/Q66710 -NCT of Delhi,wikidataId/Q1352 +NCT of Delhi,wikidataId/Q1353 Delhi,wikidataId/Q1353 Goa,wikidataId/Q1171 Gujarat,wikidataId/Q1061 @@ -38,6 +38,7 @@ Uttarakhand,wikidataId/Q1499 West Bengal,wikidataId/Q1356 ALL INDIA,country/IND All-India,country/IND +All India,country/IND Ladakh,wikidataId/Q200667 Kolkata**,wikidataId/Q1348 Chennai**,wikidataId/Q1352 diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json new file mode 100644 index 0000000000..c391694e7c --- /dev/null +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json @@ -0,0 +1,13 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted points is within the threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 7 + } + } + ] +} From 90610e8e3a325eb86adf2890c3954ca1dbe6f473 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Mon, 7 Sep 2026 10:15:31 +0000 Subject: [PATCH 2/9] Made changes suggested by Gemini-bot --- .../india_rbi_state_statistics/manifest.json | 36 +++++++++---------- .../rbi_download.py | 2 +- .../rbi_download_test.py | 20 +++++++++++ .../rbi_places_resolver.csv | 2 +- 4 files changed, 40 insertions(+), 20 deletions(-) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json index 0542cc79e7..af00739432 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json @@ -21,92 +21,92 @@ { "template_mcf": "output_files/agriculture/agriculture_output.tmcf", "cleaned_csv": "output_files/agriculture/agriculture_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/agriculture/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.tmcf", "cleaned_csv": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_forest_cover_output.tmcf", "cleaned_csv": "output_files/environment/state_wise_forest_cover_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_sdg_score_output.tmcf", "cleaned_csv": "output_files/environment/state_wise_sdg_score_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_tree_cover_output.tmcf", "cleaned_csv": "output_files/environment/state_wise_tree_cover_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/environment/sub_division_wise_annual_rainfall_output.tmcf", "cleaned_csv": "output_files/environment/sub_division_wise_annual_rainfall_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/price_and_wages/price_and_wages_output.tmcf", "cleaned_csv": "output_files/price_and_wages/price_and_wages_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/price_and_wages/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_availability_of_power_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_availability_of_power_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_installed_capacity_of_power_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_installed_capacity_of_power_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_length_of_national_highways_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_length_of_national_highways_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_length_of_roads_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_length_of_roads_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_length_of_state_highways_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_length_of_state_highways_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_per_capita_availability_of_power_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_per_capita_availability_of_power_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_power_requirement_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_power_requirement_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_railway_route_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_railway_route_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_telephones_per_100_population_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_telephones_per_100_population_output.csv", - "node_mcf": "output_files/*/*.mcf" + "node_mcf": "output_files/infrastructure/*.mcf" } ], "cron_schedule": "0 10 * * 1", diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py index b6c7c654aa..4c946586d0 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py @@ -92,7 +92,7 @@ def safe_to_numeric(val): return val for sheet_name, df in all_sheets_data.items(): - df = df.map(lambda x: str(x).replace('*', '').replace('@', '').strip()) + df = df.map(lambda x: str(x).replace('*', '').replace('@', '').strip() if pd.notna(x) else x) mask = df.eq('State/Union Territory').any(axis=1) state_positions = df[mask] == 'State/Union Territory' diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py index d7894865cd..e2405edbb9 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py @@ -66,6 +66,26 @@ def test_preprocess_files_converts_numeric_headers(self): self.assertEqual(processed.iloc[0, 3], '2017-18') self.assertEqual(processed.iloc[1, 0], 'Andhra Pradesh') + def test_preprocess_files_preserves_nan(self): + with tempfile.TemporaryDirectory() as directory: + file_path = pathlib.Path(directory) / 'source.xlsx' + initial_df = pd.DataFrame( + [['State/Union Territory', '2015*', None], + ['Andhra Pradesh', None, '30.5']]) + with pd.ExcelWriter(file_path, engine='openpyxl') as writer: + initial_df.to_excel(writer, + sheet_name='Sheet1', + index=False, + header=False) + + rbi_download.preprocess_files(directory) + + processed = pd.read_excel(file_path, + sheet_name='Sheet1', + header=None) + self.assertTrue(pd.isna(processed.iloc[0, 2])) + self.assertTrue(pd.isna(processed.iloc[1, 1])) + if __name__ == '__main__': absltest.main() diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv index 2b8594585f..7f3c28c3bd 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv @@ -8,7 +8,7 @@ Chandigarh,wikidataId/Q43433 Chhattisgarh,wikidataId/Q1168 Dadra & Nagar Haveli,wikidataId/Q46107 Daman & Diu,wikidataId/Q66710 -NCT of Delhi,wikidataId/Q1353 +NCT of Delhi,wikidataId/Q9357528 Delhi,wikidataId/Q1353 Goa,wikidataId/Q1171 Gujarat,wikidataId/Q1061 From 0ede682365749bd597d66efeb9def0fc75e3bb85 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Wed, 9 Sep 2026 11:53:28 +0000 Subject: [PATCH 3/9] Changes as per recommendation --- .../india_rbi_state_statistics/README.md | 30 +- .../india_rbi_state_statistics/configs.json | 334 ++++++++++++++++++ .../environment_pvmap.csv | 3 +- .../india_rbi_state_statistics/manifest.json | 3 +- .../rbi_download.py | 309 +++++++++++++--- .../rbi_download_test.py | 219 +++++++++++- .../rbi_places_resolver.csv | 4 +- .../india_rbi_state_statistics/run.sh | 47 ++- .../validation_config.json | 11 +- 9 files changed, 866 insertions(+), 94 deletions(-) create mode 100644 statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md index 19d09757af..96a71547f9 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md @@ -10,19 +10,22 @@ This import pipeline processes various socio-economic and agricultural statistic ## Configuration -The `rbi_download.py` script relies on a configuration file named `configs.py` to fetch the URLs, filenames, and categories for the data to be downloaded. +The `rbi_download.py` script relies on a local, version-controlled JSON configuration file named `configs.json` to specify URLs, filenames, and categories for the data to be downloaded. -Sample data of the config file: -URLS_CONFIG=[ +Sample structure of `configs.json`: +```json +{ + "URLS_CONFIG": [ { - "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/58T_xxxxxxxxxxxxx.XLSX", - "category": "agriculture", - "filename": "state_wise_pattern_of_land_use_gross_sown_area.xlsx" - }, - ... - ] + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/58T_xxxxxxxxxxxxx.XLSX", + "category": "agriculture", + "filename": "state_wise_pattern_of_land_use_gross_sown_area.xlsx" + } + ] +} +``` -This approach makes the import process semi-automatic: if the download URLs change in future RBI releases (which commonly happens due to shifting table numbers or publication structures), only this configuration file needs to be updated, rather than modifying the Python script itself. +This makes the import process **Semi-Automated**: if download URLs change in future RBI releases (due to updated publication links or revised tables), only `configs.json` needs to be updated without modifying script logic. Additionally, `rbi_download.py` employs connection pooling with `requests.Session()` + `HTTPAdapter(Retry(...))`, validates file magic bytes (`PK\x03\x04`), and atomically writes downloaded files to prevent corrupted/partial files. ## Data Acquisition and Initial Preprocessing @@ -288,3 +291,10 @@ python3 stat_var_processor.py \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output ``` + +## Validation Configuration and Thresholds + +This import uses `validation_config.json` with a 7% record deletion threshold (`DELETED_RECORDS_PERCENT: 7`). +- **Rationale & Analysis**: Up to 7% of records may be deleted or modified across historical revisions and cleanups of unmapped sub-divisions and regional reporting structures across RBI state tables. +- **Justification Document**: For detailed root cause analysis, table breakdown, and justification, see the [RBI State Statistics Deletion Threshold Justification Doc](https://docs.google.com/document/d/19k3zyNk_KR9D4dVpXdGvlXKk6AMOzoYU2I1khJSsX8w/edit?resourcekey=0-JR3lDK9eNfo3ftEV6YpGxQ&tab=t.0#heading=h.iiqodx2yoqc8). + diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json new file mode 100644 index 0000000000..52ef784515 --- /dev/null +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json @@ -0,0 +1,334 @@ +{ + "URLS_CONFIG": [ + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/53T_111220253720381F6FEB46769E9A3BA1775126F7.XLSX", + "category": "agriculture", + "filename": "state_wise_pattern_of_land_use_gross_sown_area.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/54T_111220257E02BB1F59FC46F2B936CC773BBC0874.XLSX", + "category": "agriculture", + "filename": "state_wise_pattern_of_land_use_net_sown_area.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/55T_11122025AFBC97CC9BA44E54BC31B3E58DBF2E21.XLSX", + "category": "agriculture", + "filename": "state_wise_pattern_of_land_use_gross_irrigated_area.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/56T_1112202564A24EAD0AA44A278DB069916CA8FECB.XLSX", + "category": "agriculture", + "filename": "state_wise_pattern_of_land_use_net_irrigated_area.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/57T_111220253F1BCFDCDF5047B495821F992E4D87B6.XLSX", + "category": "agriculture", + "filename": "state_wise_pattern_of_land_use_cropping_intensity.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/58T_111220250E85BAD71CD64324A6F4E28DD0A05129.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_foodgrains_rice.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/59T_1112202571AAC5A686F74224A1DF55D2B9D62D77.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_foodgrains_wheat.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/60T_111220251CE8EB785C6E40FAA8BA4EE805D01B93.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_foodgrains_coarse_cereals.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/61T_111220251C3BA74BCC3742A7BA09518C4DDD0BE5.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_foodgrains_pulses.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/62T_11122025F9D0951B74274E99A9C236A5CFAC1858.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_total_foodgrains.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/63T_1112202514B06350BA2849F98C3551060541C568.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_non_foodgrains_oilseeds.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/64T_111220251AE568D097A740F18A52EFEB30BDDA02.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_non_foodgrains_cotton_lintt.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/65T_11122025FF1A1A01BD604A1A9A63E572FF11B5B5.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_non_foodgrains_sugarcane.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/66T_1112202551DDDB3EE6624BE9B30EB1692CC0CBA6.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_non_foodgrains_raw_jute_mesta.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/67T_1112202594892EEE85FF4BAC9AAB5DC35661E62A.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_foodgrains_rice.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/68T_11122025D202C9C812754B4FBC0F25A348940AA8.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_foodgrains_wheat.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/69T_111220258B2ACEEE047E482AA3EDF0C09F7E3A86.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_foodgrains_coarse_cereals.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/70T_11122025A43EF0C6E0154357B8E404D9CF42998C.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_foodgrains_pulses.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/71T_111220252D54659F49874E689923BF46C29E01A5.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_total_foodgrains.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/72T_11122025D573A69D2317423FBCFB65F5622AF929.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_total_fruits.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/73T_11122025E35C4283EAE7430A88B1BF32B236A976.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_total_vegetables.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/74T_11122025EBF01225C48441C2A23B436CCC589BF1.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_total_fruits.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/75T_11122025C8A307B91418448DBF5B5E54997E7355.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_total_vegetables.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/76T_1112202535D900531AEA4C71B96F562B9AE59835.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_non_foodgrains_oilseeds.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/77T_11122025DA3881A5159A4760B53AC35C5BFBA8D9.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_non_foodgrains_cotton_lintt.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/78T_111220252516A0D23AA94A6B809CFB3B1953DC7B.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_non_foodgrains_sugarcane.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/79T_1112202567B7934A72A04BF988AAC0656AA9DB66.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_non_foodgrains_raw_jute_mesta.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/80T_111220251421B93FB22A4B8BAE3E52809636B603.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_rice.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/81T_11122025F795A58AD4964BFD9C1AF74F0557164E.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_wheat.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/82T_1112202534205116BBCA4010AC1792E31FADC9C9.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_coarse_cereals.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/83T_111220255A0477BFBDFD45AF93EDA200B2A5A66F.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_pulses.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/84T_11122025D106F74C0F98465DAF95013CB77F3A20.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_total_foodgrains.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/85T_111220250039FAA1932944E5B7627528B147FAFC.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_oilseeds.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/86T_111220258B6992228C504C2D84CBD88574260E87.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_cotton_lintt.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/87T_1112202547D1DA4285204EAFA0BAAA6CA47834C7.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_sugarcane.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/88T_1112202507582D4179654DC292A6311E3BDB2AB5.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_raw_jute_mesta.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/89T_11122025CC34A78CE2D64C1D880A9B4B6FA1135D.XLSX", + "category": "agriculture", + "filename": "state_wise_per_hectare_consumption_of_fertiliser_n_p_k.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/90T_1112202558892C39CD554E4098A8EB79C63D47D9.XLSX", + "category": "agriculture", + "filename": "state_wise_meat_production.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/91T_11122025A7A12D69EA1645129226BEAE5783E1A4.XLSX", + "category": "agriculture", + "filename": "state_wise_milk_production.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/92T_11122025867917E0AC5644ADB035966AAF9D220B.XLSX", + "category": "agriculture", + "filename": "state_wise_egg_production.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/93T_1112202561B6558711FE4088998165985924266E.XLSX", + "category": "agriculture", + "filename": "state_wise_fish_production.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/94T_111220254142688D61EB497BA75DAFDDB72F7160.XLSX", + "category": "agriculture", + "filename": "state_wise_wool_production.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/99T_111220251AFF6E04A39B48E499CE9CDE9E66838B.XLSX", + "category": "environment", + "filename": "state_wise_forest_cover.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/101T_11122025C61D6934B6D74891B15B91679CFE1689.XLSX", + "category": "environment", + "filename": "state_wise_tree_cover.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/102T_11122025C0C87B2909234302A1F4A52A9A42E4DF.XLSX", + "category": "environment", + "filename": "sub_division_wise_annual_rainfall.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/105T_11122025454345A0E067418D9326DD30AE67BF6F.XLSX", + "category": "environment", + "filename": "state_wise_expenditure_on_relief_on_natural_calamities.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/106T_111220251B177E8F85CA412DB39DCE90BCBD724A.XLSX", + "category": "environment", + "filename": "state_wise_sustainable_development_goals_score_SDGs.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/138T_111220252D21BC51E5A842E4B25E954391C10A41.XLSX", + "category": "infrastructure", + "filename": "state_wise_per_capita_availability_of_power.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/139T_111220256569FF1C11E248B7A8ED64D31C999776.XLSX", + "category": "infrastructure", + "filename": "state_wise_availability_of_power.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/140T_111220254D8DA0B69B444492B6E9BAF30F3395C8.XLSX", + "category": "infrastructure", + "filename": "state_wise_installed_capacity_of_power.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/141T_11122025615E98A72EA6475CAACFFD12FFC83057.XLSX", + "category": "infrastructure", + "filename": "state_wise_power_requirement.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/144T_111220250497208D070C44219818788DB5BCE35A.XLSX", + "category": "infrastructure", + "filename": "state_wise_length_of_national_highways.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/145T_111220259E033C52098E4732A6101C940DEF1222.XLSX", + "category": "infrastructure", + "filename": "state_wise_railway_route.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/146T_11122025B89449F4159B420E8305A9509F27FE82.XLSX", + "category": "infrastructure", + "filename": "state_wise_length_of_roads.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/147T_111220255C153B31AB2E4835ABEEAC91D7494BFA.XLSX", + "category": "infrastructure", + "filename": "state_wise_length_of_state_highways.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/148T_11122025D741C3CB2CC24F448DC99D907589DD71.XLSX", + "category": "infrastructure", + "filename": "state_wise_electricity_transmission_distribution_losses.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/149T_1112202538D6A609C55B4FEFBBC44BB6C5ED20F7.XLSX", + "category": "infrastructure", + "filename": "state_wise_telephones_per_100_population.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/150T_11122025C30EFE7568EA472C8AB0290288B160C0.XLSX", + "category": "infrastructure", + "filename": "state_wise_road_constructed_under_PMGSY.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/108T_111220251942D16B3BED4D73BE851D15D2329168.XLSX", + "category": "price_and_wages", + "filename": "state_wise_average_inflation_CPI_general.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/109T_111220250E067E49350B48659E5A50C3D357BB46.XLSX", + "category": "price_and_wages", + "filename": "state_wise_average_inflation_CPI_food_and_beverages.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/110T_11122025784C1BDC2482496B9E25DA1BA4B38A3D.XLSX", + "category": "price_and_wages", + "filename": "state_wise_average_inflation_CPI_fuel_and_light.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/111T_111220252F7D9704AB664C35A58770BAC73518DE.XLSX", + "category": "price_and_wages", + "filename": "state_wise_average_inflation_CPI_housing_urban.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/112T_11122025771E55EB688044CE8A85E52A901B5D22.XLSX", + "category": "price_and_wages", + "filename": "state_wise_average_daily_wage_rates_in _rural_india_men_construction_workers.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/113T_111220256FEBCC253D4A47419892925F897E2537.XLSX", + "category": "price_and_wages", + "filename": "state_wise_average_daily_wage_rates_in _rural_india_men_general_agricultural_labourers.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/114T_111220259CA566982E7F4BF38B3FDAC0D7587F2E.XLSX", + "category": "price_and_wages", + "filename": "state_wise_Average Daily Wage Rates in Rural_india_men_horticulture_workers.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/115T_1112202585CF99F21E654CED8990A734A043B3DA.XLSX", + "category": "price_and_wages", + "filename": "state_wise_average_daily_wage_rates_in_rural_india_men_non_agricultural_labourers.xlsx" + } + ] +} \ No newline at end of file diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv index 4ea5beff8a..ebd44f700d 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv @@ -6,8 +6,7 @@ Table 105: STATE-WISE EXPENDITURE ON RELIEF ON NATURAL CALAMITIES,measuredProper Table 106: STATE-WISE SUSTAINABLE DEVELOPMENT GOALS (SDGs) SCORE,measuredProperty,score,populationType,SustainableDevelopmentGoals,name,"""Sustainable Development Goals Score""",,,,,,,, State/ Union Territory,observationAbout,{Data},,,,,,,,,,,, State/Union Territory,observationAbout,{Data},,,,,,,,,,,, -"Haryana, Delhi & Chandigarh",observationAbout,dcid:wikidataId/Q1174 -"Haryana, Delhi and Chandigarh",observationAbout,dcid:wikidataId/Q1174 +"Haryana, Delhi & Chandigarh",observationAbout,country/IND 1987,observationDate,1987,value,{Number},,,,,,,,,, 1989,observationDate,1989,value,{Number},,,,,,,,,, 1991,observationDate,1991,value,{Number},,,,,,,,,, diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json index af00739432..d7f96314e7 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json @@ -15,7 +15,8 @@ "input_files/agriculture/*.xlsx", "input_files/environment/*.xlsx", "input_files/infrastructure/*.xlsx", - "input_files/price_and_wages/*.xlsx" + "input_files/price_and_wages/*.xlsx", + "counters/*.csv" ], "import_inputs": [ { diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py index 4c946586d0..7d6da73694 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py @@ -12,105 +12,302 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os -import tempfile -from absl import app, logging, flags +import pathlib +import re import sys +import tempfile +import time +from absl import app, flags, logging from google.cloud import storage import pandas as pd +import requests +from requests.adapters import HTTPAdapter +import urllib3 +from urllib3.util import Retry + +# Suppress unverified HTTPS warnings for RBI endpoints +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(os.path.join(SCRIPT_DIR, '../../../util')) INPUT_DIR = os.path.join(SCRIPT_DIR, "input_files") +DEFAULT_CONFIG_JSON = os.path.join(SCRIPT_DIR, 'configs.json') -from download_util_script import _retry_method +flags.DEFINE_string('config_file_path', DEFAULT_CONFIG_JSON, + 'Config file path (local json or gs:// path)') -flags.DEFINE_string( - 'config_file_path', - 'gs://unresolved_mcf/india_rbi/state_statistics/configs.py', - 'Config file path') +DEFAULT_HEADERS = { + 'User-Agent': ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' + 'AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/124.0.0.0 Safari/537.36'), + 'Accept': + 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Accept-Language': + 'en-US,en;q=0.9', +} -def reads_config_file(): - _FLAGS = flags.FLAGS - config_file_path = _FLAGS.config_file_path +XLSX_ZIP_SIGNATURE = b'PK\x03\x04' + + +def create_retry_session( + retries: int = 4, + backoff_factor: float = 2.0, + status_forcelist: tuple = (429, 500, 502, 503, 504), +) -> requests.Session: + """Creates a requests.Session with connection pooling, retries, and browser headers.""" + session = requests.Session() + retry_strategy = Retry( + total=retries, + backoff_factor=backoff_factor, + status_forcelist=status_forcelist, + allowed_methods=["HEAD", "GET"], + raise_on_status=False, + ) + adapter = HTTPAdapter(max_retries=retry_strategy, + pool_connections=10, + pool_maxsize=10) + session.mount("https://", adapter) + session.mount("http://", adapter) + session.headers.update(DEFAULT_HEADERS) + session.verify = False + return session + + +def _load_local_config(file_path): + if not os.path.exists(file_path): + return None try: - storage_client = storage.Client() - bucket_name = config_file_path.split('/')[2] - bucket = storage_client.bucket(bucket_name) - blob_name = '/'.join(config_file_path.split('/')[3:]) - blob = bucket.blob(blob_name) - file_contents = blob.download_as_text() - local_vars = {} - exec(file_contents, {}, local_vars) - return local_vars + with open(file_path, 'r', encoding='utf-8') as f: + return json.load(f) except Exception as e: - logging.fatal(f"Cannot extract url and related configs: {e}") + logging.warning(f"Failed to read local config file '{file_path}': {e}") + return None + -def download_files(URL_CONFIG): +def reads_config_file(): + _FLAGS = flags.FLAGS try: - for config in URL_CONFIG: - config_url = config.get("url") - category_name = config.get("category") - file_name = config.get("filename") - file_path = os.path.join(INPUT_DIR, category_name, file_name) - os.makedirs(os.path.join(INPUT_DIR, category_name), exist_ok=True) - file_response = _retry_method(config_url, headers=None, tries=3, delay=5, backoff=2) - if file_response: - with open(file_path, 'wb') as f: - f.write(file_response.content) - logging.info(f"Downloaded the file {file_name} successfully.") - except Exception as e: - logging.fatal(f"Download error: {str(e)}") + config_file_path = _FLAGS.config_file_path + except (flags.UnparsedFlagAccessError, AttributeError): + try: + config_file_path = _FLAGS['config_file_path'].value + except Exception: + config_file_path = DEFAULT_CONFIG_JSON + + # 1. If it's a local file path (or doesn't start with gs://), try loading it + if config_file_path and not config_file_path.startswith('gs://'): + config = _load_local_config(config_file_path) + if config and 'URLS_CONFIG' in config: + logging.info(f"Loaded config from local path: {config_file_path}") + return config + logging.warning( + f"Could not load config from local file '{config_file_path}'") + + # 2. If it's a GCS path, try loading from GCS + if config_file_path and config_file_path.startswith('gs://'): + try: + storage_client = storage.Client() + parts = config_file_path[5:].split('/', 1) + bucket_name = parts[0] + blob_name = parts[1] if len(parts) > 1 else '' + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + file_contents = blob.download_as_text() + if config_file_path.endswith('.json'): + return json.loads(file_contents) + local_vars = {} + exec(file_contents, {}, local_vars) + return local_vars + except Exception as e: + logging.warning( + f"Cannot extract url and related configs from GCS '{config_file_path}': {e}. " + "Falling back to local configuration.") + + # 3. Fall back to local configs.json in SCRIPT_DIR + config = _load_local_config(DEFAULT_CONFIG_JSON) + if config and 'URLS_CONFIG' in config: + logging.info( + f"Loaded fallback configuration from {DEFAULT_CONFIG_JSON}") + return config + + logging.error( + "Cannot extract url and related configs: all sources failed.") + raise RuntimeError( + "Cannot extract url and related configs: all sources failed.") + + +def download_files(URL_CONFIG, session=None, delay=0.5): + if not URL_CONFIG: + logging.warning("No URL configurations provided to download.") + return + + if session is None: + session = create_retry_session() + + for config in URL_CONFIG: + config_url = config.get("url") + category_name = config.get("category") + file_name = config.get("filename") + if not config_url or not category_name or not file_name: + logging.warning(f"Skipping incomplete config: {config}") + continue + + target_dir = os.path.join(INPUT_DIR, category_name) + os.makedirs(target_dir, exist_ok=True) + file_path = os.path.join(target_dir, file_name) + + if os.path.exists(file_path): + if os.path.getsize(file_path) > 0: + with open(file_path, 'rb') as f: + magic = f.read(4) + if magic == XLSX_ZIP_SIGNATURE: + logging.info(f"Skipping existing valid file: {file_name}") + continue + # Remove stale, empty, or corrupt file + try: + os.remove(file_path) + except OSError: + pass + + try: + logging.info(f"Attempting GET request to: {config_url}") + response = session.get(config_url, timeout=45) + + if response.status_code == 404: + logging.warning( + f"Table not found (HTTP 404) at {config_url}, skipping {file_name}" + ) + continue + + response.raise_for_status() + content = response.content + + if not content: + logging.error( + f"Empty response received for {file_name} from {config_url}" + ) + continue + + if not content.startswith(XLSX_ZIP_SIGNATURE): + snippet = content[:50].decode('utf-8', errors='replace') + logging.error( + f"Invalid file format received for {file_name} from {config_url}: " + f"expected XLSX ZIP signature, got: {snippet!r}") + continue + + # Write atomically to a temporary file, then move into place + with tempfile.NamedTemporaryFile('wb', + dir=target_dir, + delete=False) as tmp_file: + tmp_file.write(content) + temp_path = tmp_file.name + + os.replace(temp_path, file_path) + logging.info( + f"Downloaded the file {file_name} successfully ({len(content)} bytes)." + ) + + if delay > 0: + time.sleep(delay) + + except Exception as e: + logging.error( + f"Failed to download table {file_name} from {config_url}: {e}") + + +def _apply_map(df, func): + """Version-agnostic element-wise DataFrame mapping (uses df.map if available, else df.applymap).""" + if hasattr(df, 'map'): + return df.map(func) + return df.applymap(func) def preprocess_files(directory_path): if not os.path.isdir(directory_path): - logging.info(f"Error: Directory not found at '{directory_path}'") + logging.fatal(f"Error: Directory not found at '{directory_path}'") return - + xlsx_files = [f for f in os.listdir(directory_path) if f.endswith('.xlsx')] if not xlsx_files: - logging.info(f"No XLSX files found in the directory: {directory_path}") + logging.fatal( + f"No XLSX files found in the directory: {directory_path}") return - logging.info(f"Found {len(xlsx_files)} XLSX files to process in '{directory_path}'.") + logging.info( + f"Found {len(xlsx_files)} XLSX files to process in '{directory_path}'." + ) - for file_name in xlsx_files: + for file_name in sorted(xlsx_files): file_path = os.path.join(directory_path, file_name) logging.info(f"Processing file: {file_path}") try: - all_sheets_data = pd.read_excel(file_path, sheet_name=None, header=None) + all_sheets_data = pd.read_excel(file_path, + sheet_name=None, + header=None, + engine='openpyxl') + + def clean_cell(val): + if pd.isna(val): + return val + if isinstance(val, (int, float)): + return val + val_str = str(val).replace('*', '').replace('@', '').strip() + if not val_str or val_str.lower() == 'nan': + return float('nan') + return val_str + def safe_to_numeric(val): - val_str = str(val) + if pd.isna(val): + return val + if isinstance(val, (int, float)): + return val + val_str = str(val).strip() + if not val_str or val_str.lower() == 'nan': + return float('nan') if val_str.isdigit(): return int(val_str) try: float_val = float(val_str) + if float_val.is_integer(): + return int(float_val) return float_val except ValueError: return val + def is_state_header(val): + if pd.isna(val): + return False + return bool( + re.search(r'state\s*/\s*union\s*territory', str(val), + re.IGNORECASE)) + for sheet_name, df in all_sheets_data.items(): - df = df.map(lambda x: str(x).replace('*', '').replace('@', '').strip() if pd.notna(x) else x) - mask = df.eq('State/Union Territory').any(axis=1) - state_positions = df[mask] == 'State/Union Territory' - - df_num = df[mask].map(safe_to_numeric) - converted = df_num.mask(state_positions, 'State/Union Territory') - df = df.astype(object) - df.loc[mask, :] = converted + df = _apply_map(df, clean_cell) + is_state_cell = _apply_map(df, is_state_header) + mask = is_state_cell.any(axis=1) + + if mask.any(): + state_positions = is_state_cell.loc[mask] + df_num = _apply_map(df[mask], safe_to_numeric) + converted = df_num.mask(state_positions, df[mask]) + df = df.astype(object) + df.loc[mask, :] = converted all_sheets_data[sheet_name] = df with tempfile.TemporaryDirectory(dir=directory_path) as output_dir: output_path = os.path.join(output_dir, file_name) with pd.ExcelWriter(output_path, engine='openpyxl') as writer: for sheet_name, df in all_sheets_data.items(): - df.to_excel(writer, sheet_name=sheet_name, index=False, header=False) + df.to_excel(writer, + sheet_name=sheet_name, + index=False, + header=False) os.replace(output_path, file_path) except Exception as e: - logging.fatal(f"Error processing {file_name}: {e}") + logging.error(f"Error processing {file_name}: {e}") logging.info("All specified XLSX files have been processed.") @@ -120,11 +317,13 @@ def main(_): RBI_URL = configs['URLS_CONFIG'] download_files(RBI_URL) logging.info("Download process Completed successfully") - directories = ['agriculture', 'environment', 'infrastructure', 'price_and_wages'] + directories = [ + 'agriculture', 'environment', 'infrastructure', 'price_and_wages' + ] for directory in directories: preprocess_files(os.path.join(INPUT_DIR, directory)) logging.info("Pre-process Completed successfully") -if __name__ == "__main__": - app.run(main) \ No newline at end of file +if __name__ == "__main__": + app.run(main) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py index e2405edbb9..15cce4876f 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py @@ -12,14 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json +import os import pathlib +import sys import tempfile from unittest import mock -from absl.testing import absltest +from absl.testing import absltest, flagsaver import pandas as pd +import requests -from statvar_imports.database_on_indian_economy.india_rbi_state_statistics import rbi_download +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(_SCRIPT_DIR) + +import rbi_download class PreprocessFilesTest(absltest.TestCase): @@ -36,11 +43,11 @@ def test_preserves_workbook_and_original_error(self): pd.DataFrame, 'map', side_effect=ValueError('transform failed')), \ - mock.patch.object(rbi_download.logging, 'fatal') as fatal: + mock.patch.object(rbi_download.logging, 'error') as mock_error: rbi_download.preprocess_files(directory) self.assertEqual(workbook.read_bytes(), b'original workbook') - fatal.assert_called_once_with( + mock_error.assert_called_once_with( 'Error processing source.xlsx: transform failed') def test_preprocess_files_converts_numeric_headers(self): @@ -86,6 +93,210 @@ def test_preprocess_files_preserves_nan(self): self.assertTrue(pd.isna(processed.iloc[0, 2])) self.assertTrue(pd.isna(processed.iloc[1, 1])) + def test_preprocess_files_fatal_on_missing_dir(self): + with mock.patch.object(rbi_download.logging, 'fatal') as fatal: + rbi_download.preprocess_files('/non/existent/directory/path') + fatal.assert_called_once() + self.assertIn("Directory not found", fatal.call_args[0][0]) + + def test_preprocess_files_fatal_on_empty_dir(self): + with tempfile.TemporaryDirectory() as directory: + with mock.patch.object(rbi_download.logging, 'fatal') as fatal: + rbi_download.preprocess_files(directory) + fatal.assert_called_once() + self.assertIn("No XLSX files found", fatal.call_args[0][0]) + + def test_preprocess_files_handles_flexible_state_header(self): + with tempfile.TemporaryDirectory() as directory: + file_path = pathlib.Path(directory) / 'source.xlsx' + initial_df = pd.DataFrame([['State / Union Territory', '2018-19'], + ['State/Union Territory', '2019-20'], + ['Karnataka', '120.5']]) + with pd.ExcelWriter(file_path, engine='openpyxl') as writer: + initial_df.to_excel(writer, + sheet_name='Sheet1', + index=False, + header=False) + + rbi_download.preprocess_files(directory) + + processed = pd.read_excel(file_path, + sheet_name='Sheet1', + header=None) + self.assertEqual(processed.iloc[0, 0], 'State / Union Territory') + self.assertEqual(processed.iloc[1, 0], 'State/Union Territory') + self.assertEqual(processed.iloc[0, 1], '2018-19') + self.assertEqual(processed.iloc[1, 1], '2019-20') + self.assertEqual(processed.iloc[2, 0], 'Karnataka') + self.assertEqual(processed.iloc[2, 1], '120.5') + + def test_preprocess_files_avoids_literal_nan_and_preserves_numeric(self): + with tempfile.TemporaryDirectory() as directory: + file_path = pathlib.Path(directory) / 'source.xlsx' + initial_df = pd.DataFrame( + [['State/Union Territory', '2020', '2021', '2022'], + ['Bihar', 100, 200.5, ' '], ['Assam', 'nan', 'NAN', '300']]) + with pd.ExcelWriter(file_path, engine='openpyxl') as writer: + initial_df.to_excel(writer, + sheet_name='Sheet1', + index=False, + header=False) + + rbi_download.preprocess_files(directory) + + processed = pd.read_excel(file_path, + sheet_name='Sheet1', + header=None) + # Empty/whitespace and 'nan' strings become NaN, NOT literal string 'nan' + self.assertTrue(pd.isna(processed.iloc[1, 3])) + self.assertTrue(pd.isna(processed.iloc[2, 1])) + self.assertTrue(pd.isna(processed.iloc[2, 2])) + # Numeric values preserved + self.assertEqual(processed.iloc[1, 1], 100) + self.assertEqual(processed.iloc[1, 2], 200.5) + + +class ReadsConfigFileTest(absltest.TestCase): + + def test_reads_local_config_file(self): + with tempfile.NamedTemporaryFile('w', suffix='.json', + delete=False) as f: + json.dump( + { + 'URLS_CONFIG': [{ + 'url': 'http://example.com/test.xlsx', + 'category': 'test', + 'filename': 'test.xlsx' + }] + }, f) + temp_path = f.name + try: + with flagsaver.flagsaver(config_file_path=temp_path): + configs = rbi_download.reads_config_file() + self.assertIn('URLS_CONFIG', configs) + self.assertEqual(len(configs['URLS_CONFIG']), 1) + self.assertEqual(configs['URLS_CONFIG'][0]['filename'], + 'test.xlsx') + finally: + os.remove(temp_path) + + def test_falls_back_to_local_configs_json(self): + with flagsaver.flagsaver(config_file_path='gs://nonexistent_bucket/configs.json'), \ + mock.patch.object(rbi_download.storage, 'Client', side_effect=Exception('GCS unavailable')): + configs = rbi_download.reads_config_file() + self.assertIn('URLS_CONFIG', configs) + self.assertGreater(len(configs['URLS_CONFIG']), 0) + + +class DownloadFilesTest(absltest.TestCase): + + def test_skips_existing_valid_file(self): + with tempfile.TemporaryDirectory() as temp_dir: + cat_dir = os.path.join(temp_dir, 'agri') + os.makedirs(cat_dir, exist_ok=True) + existing_file = os.path.join(cat_dir, 'existing.xlsx') + with open(existing_file, 'wb') as f: + f.write(b'PK\x03\x04valid_zip_content') + + configs = [{ + 'url': 'http://example.com/existing.xlsx', + 'category': 'agri', + 'filename': 'existing.xlsx' + }] + mock_session = mock.MagicMock() + with mock.patch.object(rbi_download, 'INPUT_DIR', temp_dir): + rbi_download.download_files(configs, + session=mock_session, + delay=0) + mock_session.get.assert_not_called() + + def test_redownloads_invalid_existing_file(self): + with tempfile.TemporaryDirectory() as temp_dir: + cat_dir = os.path.join(temp_dir, 'agri') + os.makedirs(cat_dir, exist_ok=True) + corrupt_file = os.path.join(cat_dir, 'corrupt.xlsx') + with open(corrupt_file, 'wb') as f: + f.write(b'error page') + + configs = [{ + 'url': 'http://example.com/corrupt.xlsx', + 'category': 'agri', + 'filename': 'corrupt.xlsx' + }] + mock_session = mock.MagicMock() + mock_response = mock.MagicMock() + mock_response.status_code = 200 + mock_response.content = b'PK\x03\x04valid_content' + mock_session.get.return_value = mock_response + + with mock.patch.object(rbi_download, 'INPUT_DIR', temp_dir): + rbi_download.download_files(configs, + session=mock_session, + delay=0) + mock_session.get.assert_called_once() + self.assertEqual( + pathlib.Path(corrupt_file).read_bytes(), + b'PK\x03\x04valid_content') + + def test_download_files_uses_browser_headers_in_session(self): + session = rbi_download.create_retry_session() + headers = session.headers + self.assertIn('User-Agent', headers) + self.assertIn('Mozilla', headers['User-Agent']) + + def test_rejects_html_error_response(self): + with tempfile.TemporaryDirectory() as temp_dir: + configs = [{ + 'url': 'http://example.com/blocked.xlsx', + 'category': 'agri', + 'filename': 'blocked.xlsx' + }] + mock_session = mock.MagicMock() + mock_response = mock.MagicMock() + mock_response.status_code = 200 + mock_response.content = b'Access Denied' + mock_session.get.return_value = mock_response + + with mock.patch.object(rbi_download, 'INPUT_DIR', temp_dir): + rbi_download.download_files(configs, + session=mock_session, + delay=0) + target_file = pathlib.Path(temp_dir) / 'agri' / 'blocked.xlsx' + self.assertFalse(target_file.exists()) + + def test_isolates_download_failures(self): + with tempfile.TemporaryDirectory() as temp_dir: + configs = [ + { + 'url': 'http://example.com/fail.xlsx', + 'category': 'agri', + 'filename': 'fail.xlsx' + }, + { + 'url': 'http://example.com/success.xlsx', + 'category': 'agri', + 'filename': 'success.xlsx' + }, + ] + mock_session = mock.MagicMock() + mock_success = mock.MagicMock() + mock_success.status_code = 200 + mock_success.content = b'PK\x03\x04success content' + mock_session.get.side_effect = [ + requests.exceptions.RequestException('404 Not Found'), + mock_success + ] + + with mock.patch.object(rbi_download, 'INPUT_DIR', temp_dir): + rbi_download.download_files(configs, + session=mock_session, + delay=0) + + success_file = pathlib.Path(temp_dir) / 'agri' / 'success.xlsx' + self.assertTrue(success_file.exists()) + self.assertEqual(success_file.read_bytes(), + b'PK\x03\x04success content') + if __name__ == '__main__': absltest.main() diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv index 7f3c28c3bd..ed2aea2d1f 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv @@ -40,8 +40,8 @@ ALL INDIA,country/IND All-India,country/IND All India,country/IND Ladakh,wikidataId/Q200667 -Kolkata**,wikidataId/Q1348 -Chennai**,wikidataId/Q1352 +Kolkata,wikidataId/Q1348 +Chennai,wikidataId/Q1352 Mumbai,wikidataId/Q1156 TamilNadu,wikidataId/Q1445 Andaman & NicobarIslands,wikidataId/Q40888 \ No newline at end of file diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh index 29f9bdd623..f73ef6e360 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh @@ -2,43 +2,52 @@ set -e SCRIPT_PATH=$(realpath "$(dirname "$0")") +COUNTERS_PATH=$SCRIPT_PATH/counters +mkdir -p "$COUNTERS_PATH" -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/agriculture/*.xlsx --pv_map=$SCRIPT_PATH/agriculture_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/agriculture/agriculture_output || { echo "Error: Processing Agriculture files failed!"; exit 1; } +# Existing statvar MCF path with local fallback +LOCAL_MCF="$SCRIPT_PATH/stat_vars.mcf" +if [ -f "$LOCAL_MCF" ]; then + EXISTING_MCF="$LOCAL_MCF" +else + EXISTING_MCF="${EXISTING_STATVAR_MCF:-gs://unresolved_mcf/scripts/statvar/stat_vars.mcf}" +fi -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/environment/state_wise_forest_cover.xlsx --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/environment/state_wise_forest_cover_output || { echo "Error: Processing State-wise Forest Cover failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/agriculture/*.xlsx" --pv_map=$SCRIPT_PATH/agriculture_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/agriculture_counters.csv --output_path=$SCRIPT_PATH/output_files/agriculture/agriculture_output || { echo "Error: Processing Agriculture files failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/environment/state_wise_tree_cover.xlsx --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/environment/state_wise_tree_cover_output || { echo "Error: Processing State-wise Tree Cover failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/environment/state_wise_forest_cover.xlsx" --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_forest_cover_counters.csv --output_path=$SCRIPT_PATH/output_files/environment/state_wise_forest_cover_output || { echo "Error: Processing State-wise Forest Cover failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/environment/sub_division_wise_annual_rainfall.xlsx --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/environment/sub_division_wise_annual_rainfall_output || { echo "Error: Processing Sub-Division-wise Annual Rainfall failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/environment/state_wise_tree_cover.xlsx" --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_tree_cover_counters.csv --output_path=$SCRIPT_PATH/output_files/environment/state_wise_tree_cover_output || { echo "Error: Processing State-wise Tree Cover failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/environment/state_wise_expenditure_on_relief_on_natural_calamities.xlsx --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output || { echo "Error: Processing State-wise Expenditure on Relief on Natural Calamities failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/environment/sub_division_wise_annual_rainfall.xlsx" --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/sub_division_wise_annual_rainfall_counters.csv --output_path=$SCRIPT_PATH/output_files/environment/sub_division_wise_annual_rainfall_output || { echo "Error: Processing Sub-Division-wise Annual Rainfall failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/environment/state_wise_sustainable_development_goals_score_SDGs.xlsx --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/environment_sdg_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/environment/state_wise_sdg_score_output || { echo "Error: Processing State-wise Sustainable Development Goals Score failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/environment/state_wise_expenditure_on_relief_on_natural_calamities.xlsx" --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_expenditure_on_relief_on_natural_calamities_counters.csv --output_path=$SCRIPT_PATH/output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output || { echo "Error: Processing State-wise Expenditure on Relief on Natural Calamities failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/price_and_wages/*.xlsx --pv_map=$SCRIPT_PATH/price_wages_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/price_and_wages/price_and_wages_output || { echo "Error: Processing price_and_wages files failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/environment/state_wise_sustainable_development_goals_score_SDGs.xlsx" --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/environment_sdg_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_sdg_score_counters.csv --output_path=$SCRIPT_PATH/output_files/environment/state_wise_sdg_score_output || { echo "Error: Processing State-wise Sustainable Development Goals Score failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_per_capita_availability_of_power.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_per_capita_availability_of_power_output || { echo "Error: Processing State-wise Per Capita Availability of Power failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/price_and_wages/*.xlsx" --pv_map=$SCRIPT_PATH/price_wages_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/price_and_wages_counters.csv --output_path=$SCRIPT_PATH/output_files/price_and_wages/price_and_wages_output || { echo "Error: Processing price_and_wages files failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_availability_of_power.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_availability_of_power_output || { echo "Error: Processing State-wise Availability of Power files failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_per_capita_availability_of_power.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_per_capita_availability_of_power_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_per_capita_availability_of_power_output || { echo "Error: Processing State-wise Per Capita Availability of Power failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_installed_capacity_of_power.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_installed_capacity_of_power_output || { echo "Error: Processing State-wise Installed Capacity of Power failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_availability_of_power.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_availability_of_power_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_availability_of_power_output || { echo "Error: Processing State-wise Availability of Power files failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_power_requirement.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_power_requirement_output || { echo "Error: Processing State-wise Power Requirement failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_installed_capacity_of_power.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_installed_capacity_of_power_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_installed_capacity_of_power_output || { echo "Error: Processing State-wise Installed Capacity of Power failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_national_highways.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_national_highways_output || { echo "Error: Processing State-wise Length of National Highways failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_power_requirement.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_power_requirement_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_power_requirement_output || { echo "Error: Processing State-wise Power Requirement failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_railway_route.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_railway_route_output || { echo "Error: Processing State-wise Railway Route failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_national_highways.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_length_of_national_highways_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_national_highways_output || { echo "Error: Processing State-wise Length of National Highways failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_roads.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_roads_output || { echo "Error: Processing State-wise Tree Cover failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_railway_route.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_railway_route_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_railway_route_output || { echo "Error: Processing State-wise Railway Route failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_state_highways.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_state_highways_output || { echo "Error: Processing State-wise Length of State Highways failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_roads.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_length_of_roads_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_roads_output || { echo "Error: Processing State-wise Length of Roads failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_electricity_transmission_distribution_losses.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output || { echo "Error: Processing State-wise Electricity Transmission & Distribution Losses failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_state_highways.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_length_of_state_highways_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_state_highways_output || { echo "Error: Processing State-wise Length of State Highways failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_telephones_per_100_population.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_telephones_per_100_population_output || { echo "Error: Processing State-wise Telephones per 100 Population failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_electricity_transmission_distribution_losses.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_electricity_transmission_distribution_losses_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output || { echo "Error: Processing State-wise Electricity Transmission & Distribution Losses failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_road_constructed_under_PMGSY.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output || { echo "Error: Processing State-wise Road Constructed under PMGSY failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_telephones_per_100_population.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_telephones_per_100_population_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_telephones_per_100_population_output || { echo "Error: Processing State-wise Telephones per 100 Population failed!"; exit 1; } + +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_road_constructed_under_PMGSY.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_road_constructed_under_pmgsy_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output || { echo "Error: Processing State-wise Road Constructed under PMGSY failed!"; exit 1; } echo "All processing steps completed successfully." exit 0 - diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json index c391694e7c..9655dc9f61 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json @@ -3,11 +3,20 @@ "rules": [ { "rule_id": "check_deleted_records_percent", - "description": "Checks that the percentage of deleted points is within the threshold.", + "description": "Checks that the percentage of deleted records is within the 7% threshold.", "validator": "DELETED_RECORDS_PERCENT", "params": { "threshold": 7 } + }, + { + "rule_id": "check_max_date_freshness", + "description": "Verifies that each table meets its expected freshness threshold according to RBI publishing cadence.", + "validator": "SQL_VALIDATOR", + "params": { + "query": "SELECT CASE WHEN any_value(StatVar) IN ('Length_Transportation_Road', 'Length_Transportation_StateHighway') THEN MAX(MaxDate) >= '2020' WHEN any_value(StatVar) IN ('Area_LandCover_Forest', 'Score_SustainableDevelopmentGoals', 'Area_LandCover_Tree', 'Loss_Electricity_TransmissionAndDistribution_AsAFractionOf_Annual_Generation_Electricity') THEN MAX(MaxDate) >= '2023' ELSE MAX(MaxDate) >= '2024' END AS is_fresh, COUNT(*) AS row_cnt FROM stats", + "condition": "is_fresh = TRUE AND row_cnt > 0" + } } ] } From 263c3473084e698a1312692b9b77a9ca84c40e09 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Thu, 10 Sep 2026 10:58:36 +0000 Subject: [PATCH 4/9] Changes made as per comments --- .../india_rbi_state_statistics/README.md | 4 +- .../india_rbi_state_statistics/manifest.json | 14 ++---- .../rbi_download.py | 49 ++++++++++++++----- .../rbi_download_test.py | 40 +++++++++++++-- .../validation_config.json | 4 +- 5 files changed, 82 insertions(+), 29 deletions(-) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md index 96a71547f9..175b1330ac 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md @@ -294,7 +294,7 @@ python3 stat_var_processor.py \ ## Validation Configuration and Thresholds -This import uses `validation_config.json` with a 7% record deletion threshold (`DELETED_RECORDS_PERCENT: 7`). +This import uses `validation_config.json` with a 6% record deletion threshold (`DELETED_RECORDS_PERCENT: 7`). - **Rationale & Analysis**: Up to 7% of records may be deleted or modified across historical revisions and cleanups of unmapped sub-divisions and regional reporting structures across RBI state tables. -- **Justification Document**: For detailed root cause analysis, table breakdown, and justification, see the [RBI State Statistics Deletion Threshold Justification Doc](https://docs.google.com/document/d/19k3zyNk_KR9D4dVpXdGvlXKk6AMOzoYU2I1khJSsX8w/edit?resourcekey=0-JR3lDK9eNfo3ftEV6YpGxQ&tab=t.0#heading=h.iiqodx2yoqc8). +- **Justification Document**: For detailed root cause analysis, table breakdown, and justification, see the [RBI State Statistics Deletion Threshold Justification Doc](https://docs.google.com/document/d/1BLArT3T2-2EVql0Ol8tSYw9QtjFjzCzockJBquMC4AY/edit). diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json index d7f96314e7..4cc195e48b 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json @@ -26,28 +26,24 @@ }, { "template_mcf": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.csv", - "node_mcf": "output_files/environment/*.mcf" + "cleaned_csv": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.csv" }, { "template_mcf": "output_files/environment/state_wise_forest_cover_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_forest_cover_output.csv", - "node_mcf": "output_files/environment/*.mcf" + "cleaned_csv": "output_files/environment/state_wise_forest_cover_output.csv" }, { "template_mcf": "output_files/environment/state_wise_sdg_score_output.tmcf", "cleaned_csv": "output_files/environment/state_wise_sdg_score_output.csv", - "node_mcf": "output_files/environment/*.mcf" + "node_mcf": "output_files/environment/state_wise_sdg_score_output*.mcf" }, { "template_mcf": "output_files/environment/state_wise_tree_cover_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_tree_cover_output.csv", - "node_mcf": "output_files/environment/*.mcf" + "cleaned_csv": "output_files/environment/state_wise_tree_cover_output.csv" }, { "template_mcf": "output_files/environment/sub_division_wise_annual_rainfall_output.tmcf", - "cleaned_csv": "output_files/environment/sub_division_wise_annual_rainfall_output.csv", - "node_mcf": "output_files/environment/*.mcf" + "cleaned_csv": "output_files/environment/sub_division_wise_annual_rainfall_output.csv" }, { "template_mcf": "output_files/price_and_wages/price_and_wages_output.tmcf", diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py index 7d6da73694..534bcb48b3 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py @@ -24,12 +24,8 @@ import pandas as pd import requests from requests.adapters import HTTPAdapter -import urllib3 from urllib3.util import Retry -# Suppress unverified HTTPS warnings for RBI endpoints -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) INPUT_DIR = os.path.join(SCRIPT_DIR, "input_files") DEFAULT_CONFIG_JSON = os.path.join(SCRIPT_DIR, 'configs.json') @@ -70,7 +66,6 @@ def create_retry_session( session.mount("https://", adapter) session.mount("http://", adapter) session.headers.update(DEFAULT_HEADERS) - session.verify = False return session @@ -140,11 +135,13 @@ def reads_config_file(): def download_files(URL_CONFIG, session=None, delay=0.5): if not URL_CONFIG: logging.warning("No URL configurations provided to download.") - return + return [] if session is None: session = create_retry_session() + failed_downloads = [] + for config in URL_CONFIG: config_url = config.get("url") category_name = config.get("category") @@ -187,6 +184,7 @@ def download_files(URL_CONFIG, session=None, delay=0.5): logging.error( f"Empty response received for {file_name} from {config_url}" ) + failed_downloads.append(file_name) continue if not content.startswith(XLSX_ZIP_SIGNATURE): @@ -194,6 +192,7 @@ def download_files(URL_CONFIG, session=None, delay=0.5): logging.error( f"Invalid file format received for {file_name} from {config_url}: " f"expected XLSX ZIP signature, got: {snippet!r}") + failed_downloads.append(file_name) continue # Write atomically to a temporary file, then move into place @@ -214,10 +213,15 @@ def download_files(URL_CONFIG, session=None, delay=0.5): except Exception as e: logging.error( f"Failed to download table {file_name} from {config_url}: {e}") + failed_downloads.append(file_name) + + return failed_downloads def _apply_map(df, func): - """Version-agnostic element-wise DataFrame mapping (uses df.map if available, else df.applymap).""" + """Version-agnostic element-wise DataFrame mapping + (uses df.map if available, else df.applymap). + """ if hasattr(df, 'map'): return df.map(func) return df.applymap(func) @@ -226,19 +230,21 @@ def _apply_map(df, func): def preprocess_files(directory_path): if not os.path.isdir(directory_path): logging.fatal(f"Error: Directory not found at '{directory_path}'") - return + return [directory_path] xlsx_files = [f for f in os.listdir(directory_path) if f.endswith('.xlsx')] if not xlsx_files: logging.fatal( f"No XLSX files found in the directory: {directory_path}") - return + return [directory_path] logging.info( f"Found {len(xlsx_files)} XLSX files to process in '{directory_path}'." ) + failed_files = [] + for file_name in sorted(xlsx_files): file_path = os.path.join(directory_path, file_name) logging.info(f"Processing file: {file_path}") @@ -308,20 +314,41 @@ def is_state_header(val): except Exception as e: logging.error(f"Error processing {file_name}: {e}") + failed_files.append(file_name) + + if failed_files: + logging.fatal( + f"Failed to process {len(failed_files)} file(s) in '{directory_path}': {failed_files}" + ) + return failed_files logging.info("All specified XLSX files have been processed.") + return [] def main(_): configs = reads_config_file() RBI_URL = configs['URLS_CONFIG'] - download_files(RBI_URL) + failed_downloads = download_files(RBI_URL) + if failed_downloads: + logging.fatal( + f"Download failed for {len(failed_downloads)} table(s): {failed_downloads}" + ) + return logging.info("Download process Completed successfully") directories = [ 'agriculture', 'environment', 'infrastructure', 'price_and_wages' ] + failed_preprocesses = [] for directory in directories: - preprocess_files(os.path.join(INPUT_DIR, directory)) + failed = preprocess_files(os.path.join(INPUT_DIR, directory)) + if failed: + failed_preprocesses.extend(failed) + if failed_preprocesses: + logging.fatal( + f"Preprocessing failed for {len(failed_preprocesses)} file(s): {failed_preprocesses}" + ) + return logging.info("Pre-process Completed successfully") diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py index 15cce4876f..50f548fed7 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py @@ -43,12 +43,16 @@ def test_preserves_workbook_and_original_error(self): pd.DataFrame, 'map', side_effect=ValueError('transform failed')), \ - mock.patch.object(rbi_download.logging, 'error') as mock_error: - rbi_download.preprocess_files(directory) + mock.patch.object(rbi_download.logging, 'error') as mock_error, \ + mock.patch.object(rbi_download.logging, 'fatal') as mock_fatal: + failed = rbi_download.preprocess_files(directory) self.assertEqual(workbook.read_bytes(), b'original workbook') mock_error.assert_called_once_with( 'Error processing source.xlsx: transform failed') + mock_fatal.assert_called_once() + self.assertIn('source.xlsx', mock_fatal.call_args[0][0]) + self.assertEqual(failed, ['source.xlsx']) def test_preprocess_files_converts_numeric_headers(self): with tempfile.TemporaryDirectory() as directory: @@ -288,14 +292,40 @@ def test_isolates_download_failures(self): ] with mock.patch.object(rbi_download, 'INPUT_DIR', temp_dir): - rbi_download.download_files(configs, - session=mock_session, - delay=0) + failed = rbi_download.download_files(configs, + session=mock_session, + delay=0) success_file = pathlib.Path(temp_dir) / 'agri' / 'success.xlsx' self.assertTrue(success_file.exists()) self.assertEqual(success_file.read_bytes(), b'PK\x03\x04success content') + self.assertEqual(failed, ['fail.xlsx']) + + +class MainExecutionTest(absltest.TestCase): + + def test_main_fatal_when_download_fails(self): + with mock.patch.object( + rbi_download, 'reads_config_file', return_value={'URLS_CONFIG': []}), \ + mock.patch.object( + rbi_download, 'download_files', return_value=['failed_table.xlsx']), \ + mock.patch.object(rbi_download.logging, 'fatal') as mock_fatal: + rbi_download.main([]) + mock_fatal.assert_called_once() + self.assertIn('failed_table.xlsx', mock_fatal.call_args[0][0]) + + def test_main_fatal_when_preprocessing_fails(self): + with mock.patch.object( + rbi_download, 'reads_config_file', return_value={'URLS_CONFIG': []}), \ + mock.patch.object( + rbi_download, 'download_files', return_value=[]), \ + mock.patch.object( + rbi_download, 'preprocess_files', side_effect=[['broken.xlsx'], [], [], []]), \ + mock.patch.object(rbi_download.logging, 'fatal') as mock_fatal: + rbi_download.main([]) + mock_fatal.assert_called_once() + self.assertIn('broken.xlsx', mock_fatal.call_args[0][0]) if __name__ == '__main__': diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json index 9655dc9f61..a44326b54c 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json @@ -14,9 +14,9 @@ "description": "Verifies that each table meets its expected freshness threshold according to RBI publishing cadence.", "validator": "SQL_VALIDATOR", "params": { - "query": "SELECT CASE WHEN any_value(StatVar) IN ('Length_Transportation_Road', 'Length_Transportation_StateHighway') THEN MAX(MaxDate) >= '2020' WHEN any_value(StatVar) IN ('Area_LandCover_Forest', 'Score_SustainableDevelopmentGoals', 'Area_LandCover_Tree', 'Loss_Electricity_TransmissionAndDistribution_AsAFractionOf_Annual_Generation_Electricity') THEN MAX(MaxDate) >= '2023' ELSE MAX(MaxDate) >= '2024' END AS is_fresh, COUNT(*) AS row_cnt FROM stats", + "query": "SELECT StatVar, CASE WHEN StatVar IN ('Length_Transportation_Road', 'Length_Transportation_StateHighway') THEN MAX(MaxDate) >= '2020' WHEN StatVar IN ('Consumption_FarmInventory_Fertilizer') THEN MAX(MaxDate) >= '2022' WHEN StatVar IN ('Area_LandCover_Forest', 'Score_SustainableDevelopmentGoals', 'Area_LandCover_Tree', 'Loss_Electricity_TransmissionAndDistribution_AsAFractionOf_Annual_Generation_Electricity', 'GrossValue_Area_LandCover_Cropland', 'NetValue_Area_LandCover_Cropland', 'Gross_Area_LandCover_IrrigatedLand', 'NetValue_Area_LandCover_IrrigatedLand', 'GrossValue_Area_LandCover_Cropland_AsAFractionOf_NetValue_Area_LandCover_Cropland') THEN MAX(MaxDate) >= '2023' ELSE MAX(MaxDate) >= '2024' END AS is_fresh, COUNT(*) AS row_cnt FROM stats GROUP BY StatVar", "condition": "is_fresh = TRUE AND row_cnt > 0" } } ] -} +} \ No newline at end of file From 4e6de3e9a1a1b08693825aa0da18a1de7cf0b94d Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Fri, 11 Sep 2026 05:26:59 +0000 Subject: [PATCH 5/9] Made changes in downlaod script --- .../india_rbi_state_statistics/README.md | 2 +- .../india_rbi_state_statistics/manifest.json | 15 ++++--- .../rbi_download.py | 11 +++++- .../rbi_download_test.py | 39 +++++++++++++++++++ 4 files changed, 59 insertions(+), 8 deletions(-) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md index 175b1330ac..9d63f0ba7e 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md @@ -294,7 +294,7 @@ python3 stat_var_processor.py \ ## Validation Configuration and Thresholds -This import uses `validation_config.json` with a 6% record deletion threshold (`DELETED_RECORDS_PERCENT: 7`). +This import uses `validation_config.json` with a 7% record deletion threshold (`DELETED_RECORDS_PERCENT: 7`). - **Rationale & Analysis**: Up to 7% of records may be deleted or modified across historical revisions and cleanups of unmapped sub-divisions and regional reporting structures across RBI state tables. - **Justification Document**: For detailed root cause analysis, table breakdown, and justification, see the [RBI State Statistics Deletion Threshold Justification Doc](https://docs.google.com/document/d/1BLArT3T2-2EVql0Ol8tSYw9QtjFjzCzockJBquMC4AY/edit). diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json index 4cc195e48b..b0fb58dfcf 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json @@ -16,7 +16,8 @@ "input_files/environment/*.xlsx", "input_files/infrastructure/*.xlsx", "input_files/price_and_wages/*.xlsx", - "counters/*.csv" + "counters/*.csv", + "configs.json" ], "import_inputs": [ { @@ -26,11 +27,13 @@ }, { "template_mcf": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.csv" + "cleaned_csv": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.csv", + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_forest_cover_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_forest_cover_output.csv" + "cleaned_csv": "output_files/environment/state_wise_forest_cover_output.csv", + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_sdg_score_output.tmcf", @@ -39,11 +42,13 @@ }, { "template_mcf": "output_files/environment/state_wise_tree_cover_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_tree_cover_output.csv" + "cleaned_csv": "output_files/environment/state_wise_tree_cover_output.csv", + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/environment/sub_division_wise_annual_rainfall_output.tmcf", - "cleaned_csv": "output_files/environment/sub_division_wise_annual_rainfall_output.csv" + "cleaned_csv": "output_files/environment/sub_division_wise_annual_rainfall_output.csv", + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/price_and_wages/price_and_wages_output.tmcf", diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py index 534bcb48b3..d6c37e3e5c 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py @@ -111,6 +111,12 @@ def reads_config_file(): file_contents = blob.download_as_text() if config_file_path.endswith('.json'): return json.loads(file_contents) + # Deprecated: Legacy Python config (.py) via exec(). + # All imports should standardize on version-controlled JSON configuration. + logging.warning( + f"DEPRECATION WARNING: Loading Python configuration via exec() " + f"from '{config_file_path}' is deprecated and will be removed. " + "Please migrate to JSON format ('configs.json').") local_vars = {} exec(file_contents, {}, local_vars) return local_vars @@ -172,9 +178,10 @@ def download_files(URL_CONFIG, session=None, delay=0.5): response = session.get(config_url, timeout=45) if response.status_code == 404: - logging.warning( - f"Table not found (HTTP 404) at {config_url}, skipping {file_name}" + logging.error( + f"Table not found (HTTP 404) at {config_url} for {file_name}" ) + failed_downloads.append(file_name) continue response.raise_for_status() diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py index 50f548fed7..ad086458d8 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py @@ -191,6 +191,25 @@ def test_falls_back_to_local_configs_json(self): self.assertIn('URLS_CONFIG', configs) self.assertGreater(len(configs['URLS_CONFIG']), 0) + def test_reads_gcs_python_config_logs_deprecation(self): + mock_blob = mock.MagicMock() + mock_blob.download_as_text.return_value = 'URLS_CONFIG = [{"url": "http://example.com/test.xlsx", "category": "cat", "filename": "test.xlsx"}]\n' + mock_bucket = mock.MagicMock() + mock_bucket.blob.return_value = mock_blob + mock_client = mock.MagicMock() + mock_client.bucket.return_value = mock_bucket + + with flagsaver.flagsaver(config_file_path='gs://bucket/legacy_configs.py'), \ + mock.patch.object(rbi_download.storage, 'Client', return_value=mock_client), \ + mock.patch.object(rbi_download.logging, 'warning') as mock_warning: + configs = rbi_download.reads_config_file() + self.assertIn('URLS_CONFIG', configs) + deprecation_logged = any( + 'DEPRECATION WARNING' in str(call[0][0]) + for call in mock_warning.call_args_list + ) + self.assertTrue(deprecation_logged) + class DownloadFilesTest(absltest.TestCase): @@ -268,6 +287,26 @@ def test_rejects_html_error_response(self): target_file = pathlib.Path(temp_dir) / 'agri' / 'blocked.xlsx' self.assertFalse(target_file.exists()) + def test_treats_404_as_failure(self): + with tempfile.TemporaryDirectory() as temp_dir: + configs = [{ + 'url': 'http://example.com/missing.xlsx', + 'category': 'agri', + 'filename': 'missing.xlsx' + }] + mock_session = mock.MagicMock() + mock_response = mock.MagicMock() + mock_response.status_code = 404 + mock_session.get.return_value = mock_response + + with mock.patch.object(rbi_download, 'INPUT_DIR', temp_dir): + failed = rbi_download.download_files(configs, + session=mock_session, + delay=0) + self.assertEqual(failed, ['missing.xlsx']) + target_file = pathlib.Path(temp_dir) / 'agri' / 'missing.xlsx' + self.assertFalse(target_file.exists()) + def test_isolates_download_failures(self): with tempfile.TemporaryDirectory() as temp_dir: configs = [ From fa01b14f2ec536f8d403be09e2b7ff2da41e5838 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Fri, 11 Sep 2026 06:44:26 +0000 Subject: [PATCH 6/9] Modified naming in config.json --- .../india_rbi_state_statistics/configs.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json index 52ef784515..11f90b347e 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json @@ -313,17 +313,17 @@ { "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/112T_11122025771E55EB688044CE8A85E52A901B5D22.XLSX", "category": "price_and_wages", - "filename": "state_wise_average_daily_wage_rates_in _rural_india_men_construction_workers.xlsx" + "filename": "state_wise_average_daily_wage_rates_in_rural_india_men_construction_workers.xlsx" }, { "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/113T_111220256FEBCC253D4A47419892925F897E2537.XLSX", "category": "price_and_wages", - "filename": "state_wise_average_daily_wage_rates_in _rural_india_men_general_agricultural_labourers.xlsx" + "filename": "state_wise_average_daily_wage_rates_in_rural_india_men_general_agricultural_labourers.xlsx" }, { "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/114T_111220259CA566982E7F4BF38B3FDAC0D7587F2E.XLSX", "category": "price_and_wages", - "filename": "state_wise_Average Daily Wage Rates in Rural_india_men_horticulture_workers.xlsx" + "filename": "state_wise_Average_daily_wage_rates_in_rural_india_men_horticulture_workers.xlsx" }, { "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/115T_1112202585CF99F21E654CED8990A734A043B3DA.XLSX", From 26ae732fc243642ef30bf7a331429434cc6cdca2 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Tue, 15 Sep 2026 08:04:37 +0000 Subject: [PATCH 7/9] Resolved comments --- .../environment_pvmap.csv | 2 +- .../india_rbi_state_statistics/rbi_download.py | 14 ++++++++++++++ .../rbi_download_test.py | 2 ++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv index ebd44f700d..8afbad64f9 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv @@ -6,7 +6,7 @@ Table 105: STATE-WISE EXPENDITURE ON RELIEF ON NATURAL CALAMITIES,measuredProper Table 106: STATE-WISE SUSTAINABLE DEVELOPMENT GOALS (SDGs) SCORE,measuredProperty,score,populationType,SustainableDevelopmentGoals,name,"""Sustainable Development Goals Score""",,,,,,,, State/ Union Territory,observationAbout,{Data},,,,,,,,,,,, State/Union Territory,observationAbout,{Data},,,,,,,,,,,, -"Haryana, Delhi & Chandigarh",observationAbout,country/IND +"Haryana, Delhi & Chandigarh",observationAbout,dcid:HaryanaDelhiChandigarh 1987,observationDate,1987,value,{Number},,,,,,,,,, 1989,observationDate,1989,value,{Number},,,,,,,,,, 1991,observationDate,1991,value,{Number},,,,,,,,,, diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py index d6c37e3e5c..74ad581287 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py @@ -41,8 +41,14 @@ 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.9', + 'Referer': + 'https://www.rbi.org.in/', } +RBI_PORTAL_URL = ( + 'https://www.rbi.org.in/Scripts/AnnualPublications.aspx?' + 'head=Handbook%20of%20Statistics%20on%20Indian%20States' +) XLSX_ZIP_SIGNATURE = b'PK\x03\x04' @@ -145,6 +151,14 @@ def download_files(URL_CONFIG, session=None, delay=0.5): if session is None: session = create_retry_session() + try: + logging.info( + f"Warming up session with RBI portal cookies from {RBI_PORTAL_URL}" + ) + session.get(RBI_PORTAL_URL, timeout=30) + except Exception as e: + logging.warning( + f"Could not warm up session from {RBI_PORTAL_URL}: {e}") failed_downloads = [] diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py index ad086458d8..1b9b229020 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py @@ -266,6 +266,8 @@ def test_download_files_uses_browser_headers_in_session(self): headers = session.headers self.assertIn('User-Agent', headers) self.assertIn('Mozilla', headers['User-Agent']) + self.assertIn('Referer', headers) + self.assertIn('rbi.org.in', headers['Referer']) def test_rejects_html_error_response(self): with tempfile.TemporaryDirectory() as temp_dir: From 9d291ed728514de0e5a1add2c6fa8fd328f8cd73 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Thu, 17 Sep 2026 08:07:08 +0000 Subject: [PATCH 8/9] CRA analysis done --- .../india_rbi_state_statistics/README.md | 102 ++++++++++++------ .../india_rbi_state_statistics/manifest.json | 34 +++--- .../rbi_download.py | 6 +- .../india_rbi_state_statistics/run.sh | 3 +- .../validation_config.json | 11 +- 5 files changed, 103 insertions(+), 53 deletions(-) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md index 9d63f0ba7e..e56ba2a9db 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md @@ -29,14 +29,16 @@ This makes the import process **Semi-Automated**: if download URLs change in fut ## Data Acquisition and Initial Preprocessing -The `rbi_download.py` script is responsible for both downloading the raw Excel data and performing an initial preprocessing step. This preprocessing involves reading all sheets from the downloaded `.xlsx` files and replacing any `*` symbols with empty values. +The `rbi_download.py` script is responsible for both downloading the raw Excel data and performing an initial preprocessing step: +- **Cell Cleaning (`clean_cell`)**: Strips footnote markers (`*` and `@`), trims leading/trailing whitespace, and converts empty or literal `'nan'` strings to true `NaN` values so `stat_var_processor.py` does not emit malformed string observations. +- **Header Normalization (`safe_to_numeric`)**: Converts numeric or float-coerced year headers (e.g., `2015.0`, `'2016@'`) in `State/Union Territory` header rows to integers (`2015`, `2016`) so they match integer string keys in the PVMaps. ### How to Run: Execute the `rbi_download.py` script. This script will: 1. Automatically create an `input_files` directory if it doesn't exist. -2. Download the necessary Excel files into subfolders within `input_files` (e.g., `input_files/agriculture/`). -3. Process each downloaded Excel file by reading all its sheets and replacing `*` characters with empty strings in all cells. +2. Download the necessary Excel files into subfolders within `input_files` (e.g., `input_files/agriculture/`), verifying `PK\x03\x04` ZIP signatures and writing files atomically. +3. Process each downloaded Excel file across all sheets using `clean_cell` and `safe_to_numeric`. ### Download Command: @@ -44,20 +46,32 @@ Execute the `rbi_download.py` script. This script will: python3 rbi_download.py ``` +Optional flag: +- `--config_file_path`: Path to the JSON config file (defaults to local `configs.json`; also supports a GCS URI such as `gs:///configs.json`). + +### Running Unit Tests: + +To run the hermetic unit test suite for `rbi_download.py`: + +```bash +python3 rbi_download_test.py +``` + ## Processing Section -The downloaded data is processed using the `stat_var_processor.py` script, which is part of the `/data/tools/statvar_importer/` toolkit. This script converts the raw Excel data into a structured format suitable for further analysis and ingestion. Each processing command specifies the input data file(s), the Property-Value (PV) map, the configuration file (metadata), a places resolver CSV, and the desired output path. +The downloaded data is processed using the `stat_var_processor.py` script, which is part of the `/data/tools/statvar_importer/` toolkit. This script converts the raw Excel data into a structured format suitable for further analysis and ingestion. Each processing command specifies the input data file(s), the Property-Value (PV) map, the configuration file (metadata), a places resolver CSV, counter output CSV, and the desired output path. ### General Processing Command Structure: ```bash python3 stat_var_processor.py \ - --input_data='.xlsx' \ - --pv_map='' \ - --config_file='' \ - --places_resolved_csv='' \ - --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ - --output_path='' + --input_data=".xlsx" \ + --pv_map="" \ + --config_file="" \ + --places_resolved_csv="" \ + --existing_statvar_mcf="gs://unresolved_mcf/scripts/statvar/stat_vars.mcf" \ + --output_counters="" \ + --output_path="" ``` ### How to Run Processing: @@ -72,7 +86,7 @@ sh run.sh #### Option 2: Executing Individual Processing Commands -Navigate to the `/data/tools/statvar_importer/` directory before running the following commands, or adjust the relative paths accordingly. +Navigate to the `/data/tools/statvar_importer/` directory before running the following commands, or adjust the relative paths accordingly. Note that wildcard `--input_data` paths must be enclosed in double quotes so `stat_var_processor.py` expands them internally. ### Processing Commands: @@ -80,11 +94,12 @@ Navigate to the `/data/tools/statvar_importer/` directory before running the fol ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/agriculture/*.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/agriculture/*.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/agriculture_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/agriculture_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/agriculture/agriculture_output ``` @@ -92,11 +107,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/environment/state_wise_forest_cover.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/environment/state_wise_forest_cover.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_forest_cover_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/environment/state_wise_forest_cover_output ``` @@ -104,11 +120,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/environment/state_wise_tree_cover.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/environment/state_wise_tree_cover.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_tree_cover_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/environment/state_wise_tree_cover_output ``` @@ -116,11 +133,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/environment/sub_division_wise_annual_rainfall.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/environment/sub_division_wise_annual_rainfall.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/sub_division_wise_annual_rainfall_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/environment/sub_division_wise_annual_rainfall_output ``` @@ -128,11 +146,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/environment/state_wise_expenditure_on_relief_on_natural_calamities.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/environment/state_wise_expenditure_on_relief_on_natural_calamities.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_expenditure_on_relief_on_natural_calamities_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output ``` @@ -140,11 +159,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/environment/state_wise_sustainable_development_goals_score_SDGs.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/environment/state_wise_sustainable_development_goals_score_SDGs.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_sdg_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_sdg_score_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/environment/state_wise_sdg_score_output ``` @@ -152,11 +172,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/price_and_wages/*.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/price_and_wages/*.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/price_wages_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/price_and_wages_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/price_and_wages/price_and_wages_output ``` @@ -164,11 +185,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_per_capita_availability_of_power.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_per_capita_availability_of_power.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_per_capita_availability_of_power_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_per_capita_availability_of_power_output ``` @@ -176,11 +198,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_availability_of_power.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_availability_of_power.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_availability_of_power_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_availability_of_power_output ``` @@ -188,11 +211,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_installed_capacity_of_power.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_installed_capacity_of_power.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_installed_capacity_of_power_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_installed_capacity_of_power_output ``` @@ -200,11 +224,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_power_requirement.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_power_requirement.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_power_requirement_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_power_requirement_output ``` @@ -212,11 +237,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_length_of_national_highways.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_length_of_national_highways.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_length_of_national_highways_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_length_of_national_highways_output ``` @@ -224,11 +250,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_railway_route.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_railway_route.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_railway_route_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_railway_route_output ``` @@ -236,11 +263,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_length_of_roads.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_length_of_roads.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_length_of_roads_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_length_of_roads_output ``` @@ -248,11 +276,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_length_of_state_highways.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_length_of_state_highways.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_length_of_state_highways_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_length_of_state_highways_output ``` @@ -260,11 +289,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_electricity_transmission_distribution_losses.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_electricity_transmission_distribution_losses.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_electricity_transmission_distribution_losses_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output ``` @@ -272,11 +302,12 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_telephones_per_100_population.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_telephones_per_100_population.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_telephones_per_100_population_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_telephones_per_100_population_output ``` @@ -284,17 +315,24 @@ python3 stat_var_processor.py \ ```bash python3 stat_var_processor.py \ - --input_data=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_road_constructed_under_PMGSY.xlsx \ + --input_data="../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/input_files/infrastructure/state_wise_road_constructed_under_PMGSY.xlsx" \ --pv_map=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_pvmap.csv \ --config_file=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv \ --places_resolved_csv=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_counters=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/counters/state_wise_road_constructed_under_pmgsy_counters.csv \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output ``` ## Validation Configuration and Thresholds -This import uses `validation_config.json` with a 7% record deletion threshold (`DELETED_RECORDS_PERCENT: 7`). -- **Rationale & Analysis**: Up to 7% of records may be deleted or modified across historical revisions and cleanups of unmapped sub-divisions and regional reporting structures across RBI state tables. -- **Justification Document**: For detailed root cause analysis, table breakdown, and justification, see the [RBI State Statistics Deletion Threshold Justification Doc](https://docs.google.com/document/d/1BLArT3T2-2EVql0Ol8tSYw9QtjFjzCzockJBquMC4AY/edit). +This import uses `validation_config.json` with: +1. **`DELETED_RECORDS_PERCENT: 7` (`check_deleted_records_percent`)**: + - **Rationale & Analysis**: Up to 7% of records may be deleted or modified across historical revisions and cleanups of unmapped sub-divisions and regional reporting structures across RBI state tables. + - **Justification Document**: For detailed root cause analysis, table breakdown, and justification, see the [RBI State Statistics Deletion Threshold Justification Doc](https://docs.google.com/document/d/1BLArT3T2-2EVql0Ol8tSYw9QtjFjzCzockJBquMC4AY/edit). +2. **`SQL_VALIDATOR` non-empty check (`check_expected_statvar_count`)**: + - Asserts `statvar_cnt >= 1` so an empty `summary_report.csv` cannot vacuously pass grouped SQL queries. +3. **`SQL_VALIDATOR` cadence freshness check (`check_max_date_freshness`)**: + - Asserts that each StatVar meets its expected RBI publication cadence (`2020`, `2022`, `2023`, or `2024`) with `WHERE MaxDate IS NOT NULL`. + diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json index b0fb58dfcf..7b14fed2a4 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json @@ -23,17 +23,17 @@ { "template_mcf": "output_files/agriculture/agriculture_output.tmcf", "cleaned_csv": "output_files/agriculture/agriculture_output.csv", - "node_mcf": "output_files/agriculture/*.mcf" + "node_mcf": "output_files/agriculture/agriculture_output*.mcf" }, { "template_mcf": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.tmcf", "cleaned_csv": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.csv", - "node_mcf": "output_files/environment/*.mcf" + "node_mcf": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output*.mcf" }, { "template_mcf": "output_files/environment/state_wise_forest_cover_output.tmcf", "cleaned_csv": "output_files/environment/state_wise_forest_cover_output.csv", - "node_mcf": "output_files/environment/*.mcf" + "node_mcf": "output_files/environment/state_wise_forest_cover_output*.mcf" }, { "template_mcf": "output_files/environment/state_wise_sdg_score_output.tmcf", @@ -43,72 +43,72 @@ { "template_mcf": "output_files/environment/state_wise_tree_cover_output.tmcf", "cleaned_csv": "output_files/environment/state_wise_tree_cover_output.csv", - "node_mcf": "output_files/environment/*.mcf" + "node_mcf": "output_files/environment/state_wise_tree_cover_output*.mcf" }, { "template_mcf": "output_files/environment/sub_division_wise_annual_rainfall_output.tmcf", "cleaned_csv": "output_files/environment/sub_division_wise_annual_rainfall_output.csv", - "node_mcf": "output_files/environment/*.mcf" + "node_mcf": "output_files/environment/sub_division_wise_annual_rainfall_output*.mcf" }, { "template_mcf": "output_files/price_and_wages/price_and_wages_output.tmcf", "cleaned_csv": "output_files/price_and_wages/price_and_wages_output.csv", - "node_mcf": "output_files/price_and_wages/*.mcf" + "node_mcf": "output_files/price_and_wages/price_and_wages_output*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_availability_of_power_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_availability_of_power_output.csv", - "node_mcf": "output_files/infrastructure/*.mcf" + "node_mcf": "output_files/infrastructure/state_wise_availability_of_power_output*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output.csv", - "node_mcf": "output_files/infrastructure/*.mcf" + "node_mcf": "output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_installed_capacity_of_power_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_installed_capacity_of_power_output.csv", - "node_mcf": "output_files/infrastructure/*.mcf" + "node_mcf": "output_files/infrastructure/state_wise_installed_capacity_of_power_output*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_length_of_national_highways_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_length_of_national_highways_output.csv", - "node_mcf": "output_files/infrastructure/*.mcf" + "node_mcf": "output_files/infrastructure/state_wise_length_of_national_highways_output*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_length_of_roads_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_length_of_roads_output.csv", - "node_mcf": "output_files/infrastructure/*.mcf" + "node_mcf": "output_files/infrastructure/state_wise_length_of_roads_output*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_length_of_state_highways_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_length_of_state_highways_output.csv", - "node_mcf": "output_files/infrastructure/*.mcf" + "node_mcf": "output_files/infrastructure/state_wise_length_of_state_highways_output*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_per_capita_availability_of_power_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_per_capita_availability_of_power_output.csv", - "node_mcf": "output_files/infrastructure/*.mcf" + "node_mcf": "output_files/infrastructure/state_wise_per_capita_availability_of_power_output*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_power_requirement_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_power_requirement_output.csv", - "node_mcf": "output_files/infrastructure/*.mcf" + "node_mcf": "output_files/infrastructure/state_wise_power_requirement_output*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_railway_route_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_railway_route_output.csv", - "node_mcf": "output_files/infrastructure/*.mcf" + "node_mcf": "output_files/infrastructure/state_wise_railway_route_output*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output.csv", - "node_mcf": "output_files/infrastructure/*.mcf" + "node_mcf": "output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_telephones_per_100_population_output.tmcf", "cleaned_csv": "output_files/infrastructure/state_wise_telephones_per_100_population_output.csv", - "node_mcf": "output_files/infrastructure/*.mcf" + "node_mcf": "output_files/infrastructure/state_wise_telephones_per_100_population_output*.mcf" } ], "cron_schedule": "0 10 * * 1", diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py index 74ad581287..2868fb801a 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py @@ -14,9 +14,7 @@ import json import os -import pathlib import re -import sys import tempfile import time from absl import app, flags, logging @@ -72,6 +70,7 @@ def create_retry_session( session.mount("https://", adapter) session.mount("http://", adapter) session.headers.update(DEFAULT_HEADERS) + session.verify = True return session @@ -332,6 +331,9 @@ def is_state_header(val): index=False, header=False) os.replace(output_path, file_path) + logging.info( + f"Successfully processed {file_name} ({len(all_sheets_data)} sheet(s))." + ) except Exception as e: logging.error(f"Error processing {file_name}: {e}") diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh index f73ef6e360..5415a0d12b 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh @@ -49,5 +49,6 @@ python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --inp python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_road_constructed_under_PMGSY.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_road_constructed_under_pmgsy_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output || { echo "Error: Processing State-wise Road Constructed under PMGSY failed!"; exit 1; } +EXIT_STATUS=$? echo "All processing steps completed successfully." -exit 0 +exit $EXIT_STATUS diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json index a44326b54c..dfed4dc110 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json @@ -9,12 +9,21 @@ "threshold": 7 } }, + { + "rule_id": "check_expected_statvar_count", + "description": "Verifies that summary_report.csv is non-empty and contains at least one StatVar.", + "validator": "SQL_VALIDATOR", + "params": { + "query": "SELECT COUNT(DISTINCT StatVar) AS statvar_cnt FROM stats", + "condition": "statvar_cnt >= 1" + } + }, { "rule_id": "check_max_date_freshness", "description": "Verifies that each table meets its expected freshness threshold according to RBI publishing cadence.", "validator": "SQL_VALIDATOR", "params": { - "query": "SELECT StatVar, CASE WHEN StatVar IN ('Length_Transportation_Road', 'Length_Transportation_StateHighway') THEN MAX(MaxDate) >= '2020' WHEN StatVar IN ('Consumption_FarmInventory_Fertilizer') THEN MAX(MaxDate) >= '2022' WHEN StatVar IN ('Area_LandCover_Forest', 'Score_SustainableDevelopmentGoals', 'Area_LandCover_Tree', 'Loss_Electricity_TransmissionAndDistribution_AsAFractionOf_Annual_Generation_Electricity', 'GrossValue_Area_LandCover_Cropland', 'NetValue_Area_LandCover_Cropland', 'Gross_Area_LandCover_IrrigatedLand', 'NetValue_Area_LandCover_IrrigatedLand', 'GrossValue_Area_LandCover_Cropland_AsAFractionOf_NetValue_Area_LandCover_Cropland') THEN MAX(MaxDate) >= '2023' ELSE MAX(MaxDate) >= '2024' END AS is_fresh, COUNT(*) AS row_cnt FROM stats GROUP BY StatVar", + "query": "SELECT StatVar, CASE WHEN StatVar IN ('Length_Transportation_Road', 'Length_Transportation_StateHighway') THEN MAX(MaxDate) >= '2020' WHEN StatVar IN ('Consumption_FarmInventory_Fertilizer') THEN MAX(MaxDate) >= '2022' WHEN StatVar IN ('Area_LandCover_Forest', 'Score_SustainableDevelopmentGoals', 'Area_LandCover_Tree', 'Loss_Electricity_TransmissionAndDistribution_AsAFractionOf_Annual_Generation_Electricity', 'GrossValue_Area_LandCover_Cropland', 'NetValue_Area_LandCover_Cropland', 'Gross_Area_LandCover_IrrigatedLand', 'NetValue_Area_LandCover_IrrigatedLand', 'GrossValue_Area_LandCover_Cropland_AsAFractionOf_NetValue_Area_LandCover_Cropland') THEN MAX(MaxDate) >= '2023' ELSE MAX(MaxDate) >= '2024' END AS is_fresh, COUNT(*) AS row_cnt FROM stats WHERE MaxDate IS NOT NULL GROUP BY StatVar", "condition": "is_fresh = TRUE AND row_cnt > 0" } } From aed61388befbcd66ac9a5f0ce66a7bf3bb27dd77 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Thu, 17 Sep 2026 14:15:23 +0000 Subject: [PATCH 9/9] Changes as per Internal reviews --- .../india_rbi_state_statistics/README.md | 5 +- .../india_rbi_state_statistics/configs.json | 2 +- .../india_rbi_state_statistics/manifest.json | 5 +- .../rbi_download.py | 18 +++++-- .../rbi_download_test.py | 51 ++++++++++++------- .../rbi_places_resolver.csv | 2 +- .../validation_config.json | 4 +- 7 files changed, 56 insertions(+), 31 deletions(-) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md index e56ba2a9db..a4bb6e46aa 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md @@ -328,7 +328,10 @@ python3 stat_var_processor.py \ This import uses `validation_config.json` with: 1. **`DELETED_RECORDS_PERCENT: 7` (`check_deleted_records_percent`)**: - - **Rationale & Analysis**: Up to 7% of records may be deleted or modified across historical revisions and cleanups of unmapped sub-divisions and regional reporting structures across RBI state tables. + - **Rationale & Analysis**: The threshold is set to `7%` to accommodate up to **6.94%** reported deletions across the 18 sub-imports (specifically **6.94%** in `sub_division_wise_annual_rainfall_output.csv`, **5.64%** in `agriculture_output.csv`, and **3.15%** in `state_wise_expenditure_on_relief_on_natural_calamities_output.csv`). These deletions are not data loss, but result from: + 1. **Place Resolution Corrections**: Remapping `"Haryana, Delhi & Chandigarh"` from `dcid:wikidataId/Q1174` (Haryana) to the dedicated IMD sub-division DCID `dcid:HaryanaDelhiChandigarh` (24 records / 6.94% in rainfall), and fixing `NCT of Delhi` from `wikidataId/Q1352` (erroneously Chennai) to `wikidataId/Q9357528` (439 records in agriculture, 13 records in relief expenditure). + 2. **Unit Schema Corrections**: Changing `Annual_Amount_FarmInventory_Fruits` in `agriculture_pvmap.csv` from `Hectare` to `MetricTon` (309 records in agriculture, where Import Differ classifies old unit nodes as deleted and new unit nodes as added). + 3. **Official RBI Source Revisions**: Retrospective updates by RBI to provisional figures for recent fiscal years (`2022-03` through `2024-03`) and UT consolidations (`Dadra & Nagar Haveli and Daman & Diu`). - **Justification Document**: For detailed root cause analysis, table breakdown, and justification, see the [RBI State Statistics Deletion Threshold Justification Doc](https://docs.google.com/document/d/1BLArT3T2-2EVql0Ol8tSYw9QtjFjzCzockJBquMC4AY/edit). 2. **`SQL_VALIDATOR` non-empty check (`check_expected_statvar_count`)**: - Asserts `statvar_cnt >= 1` so an empty `summary_report.csv` cannot vacuously pass grouped SQL queries. diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json index 11f90b347e..023961b31b 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json @@ -331,4 +331,4 @@ "filename": "state_wise_average_daily_wage_rates_in_rural_india_men_non_agricultural_labourers.xlsx" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json index 7b14fed2a4..9642adcdb7 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json @@ -17,7 +17,8 @@ "input_files/infrastructure/*.xlsx", "input_files/price_and_wages/*.xlsx", "counters/*.csv", - "configs.json" + "configs.json", + "validation_config.json" ], "import_inputs": [ { @@ -115,4 +116,4 @@ "validation_config_file": "validation_config.json" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py index 2868fb801a..2db01a66af 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py @@ -30,6 +30,8 @@ flags.DEFINE_string('config_file_path', DEFAULT_CONFIG_JSON, 'Config file path (local json or gs:// path)') +flags.DEFINE_bool('force_download', False, + 'If true, redownload files even if valid local copies exist.') DEFAULT_HEADERS = { 'User-Agent': ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' @@ -173,14 +175,20 @@ def download_files(URL_CONFIG, session=None, delay=0.5): os.makedirs(target_dir, exist_ok=True) file_path = os.path.join(target_dir, file_name) + force_download = False + try: + force_download = flags.FLAGS.force_download + except (flags.UnparsedFlagAccessError, AttributeError): + pass + if os.path.exists(file_path): - if os.path.getsize(file_path) > 0: + if not force_download and os.path.getsize(file_path) > 0: with open(file_path, 'rb') as f: magic = f.read(4) if magic == XLSX_ZIP_SIGNATURE: logging.info(f"Skipping existing valid file: {file_name}") continue - # Remove stale, empty, or corrupt file + # Remove stale, empty, or corrupt file (or existing file if force_download) try: os.remove(file_path) except OSError: @@ -249,13 +257,13 @@ def _apply_map(df, func): def preprocess_files(directory_path): if not os.path.isdir(directory_path): - logging.fatal(f"Error: Directory not found at '{directory_path}'") + logging.error(f"Error: Directory not found at '{directory_path}'") return [directory_path] xlsx_files = [f for f in os.listdir(directory_path) if f.endswith('.xlsx')] if not xlsx_files: - logging.fatal( + logging.error( f"No XLSX files found in the directory: {directory_path}") return [directory_path] @@ -340,7 +348,7 @@ def is_state_header(val): failed_files.append(file_name) if failed_files: - logging.fatal( + logging.error( f"Failed to process {len(failed_files)} file(s) in '{directory_path}': {failed_files}" ) return failed_files diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py index 1b9b229020..f672bb7423 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py @@ -48,10 +48,10 @@ def test_preserves_workbook_and_original_error(self): failed = rbi_download.preprocess_files(directory) self.assertEqual(workbook.read_bytes(), b'original workbook') - mock_error.assert_called_once_with( + mock_error.assert_any_call( 'Error processing source.xlsx: transform failed') - mock_fatal.assert_called_once() - self.assertIn('source.xlsx', mock_fatal.call_args[0][0]) + self.assertEqual(mock_error.call_count, 2) + mock_fatal.assert_not_called() self.assertEqual(failed, ['source.xlsx']) def test_preprocess_files_converts_numeric_headers(self): @@ -97,18 +97,21 @@ def test_preprocess_files_preserves_nan(self): self.assertTrue(pd.isna(processed.iloc[0, 2])) self.assertTrue(pd.isna(processed.iloc[1, 1])) - def test_preprocess_files_fatal_on_missing_dir(self): - with mock.patch.object(rbi_download.logging, 'fatal') as fatal: - rbi_download.preprocess_files('/non/existent/directory/path') - fatal.assert_called_once() - self.assertIn("Directory not found", fatal.call_args[0][0]) + def test_preprocess_files_error_on_missing_dir(self): + with mock.patch.object(rbi_download.logging, 'error') as mock_error: + failed = rbi_download.preprocess_files( + '/non/existent/directory/path') + self.assertEqual(failed, ['/non/existent/directory/path']) + mock_error.assert_called_once() + self.assertIn("Directory not found", mock_error.call_args[0][0]) - def test_preprocess_files_fatal_on_empty_dir(self): + def test_preprocess_files_error_on_empty_dir(self): with tempfile.TemporaryDirectory() as directory: - with mock.patch.object(rbi_download.logging, 'fatal') as fatal: - rbi_download.preprocess_files(directory) - fatal.assert_called_once() - self.assertIn("No XLSX files found", fatal.call_args[0][0]) + with mock.patch.object(rbi_download.logging, 'error') as mock_error: + failed = rbi_download.preprocess_files(directory) + self.assertEqual(failed, [directory]) + mock_error.assert_called_once() + self.assertIn("No XLSX files found", mock_error.call_args[0][0]) def test_preprocess_files_handles_flexible_state_header(self): with tempfile.TemporaryDirectory() as directory: @@ -185,23 +188,33 @@ def test_reads_local_config_file(self): os.remove(temp_path) def test_falls_back_to_local_configs_json(self): - with flagsaver.flagsaver(config_file_path='gs://nonexistent_bucket/configs.json'), \ - mock.patch.object(rbi_download.storage, 'Client', side_effect=Exception('GCS unavailable')): + with flagsaver.flagsaver( + config_file_path='gs://nonexistent_bucket/configs.json'), \ + mock.patch.object( + rbi_download.storage, + 'Client', + side_effect=Exception('GCS unavailable')): configs = rbi_download.reads_config_file() self.assertIn('URLS_CONFIG', configs) self.assertGreater(len(configs['URLS_CONFIG']), 0) def test_reads_gcs_python_config_logs_deprecation(self): mock_blob = mock.MagicMock() - mock_blob.download_as_text.return_value = 'URLS_CONFIG = [{"url": "http://example.com/test.xlsx", "category": "cat", "filename": "test.xlsx"}]\n' + mock_blob.download_as_text.return_value = ( + 'URLS_CONFIG = [{"url": "http://example.com/test.xlsx", ' + '"category": "cat", "filename": "test.xlsx"}]\n' + ) mock_bucket = mock.MagicMock() mock_bucket.blob.return_value = mock_blob mock_client = mock.MagicMock() mock_client.bucket.return_value = mock_bucket - with flagsaver.flagsaver(config_file_path='gs://bucket/legacy_configs.py'), \ - mock.patch.object(rbi_download.storage, 'Client', return_value=mock_client), \ - mock.patch.object(rbi_download.logging, 'warning') as mock_warning: + with flagsaver.flagsaver( + config_file_path='gs://bucket/legacy_configs.py'), \ + mock.patch.object( + rbi_download.storage, 'Client', return_value=mock_client), \ + mock.patch.object( + rbi_download.logging, 'warning') as mock_warning: configs = rbi_download.reads_config_file() self.assertIn('URLS_CONFIG', configs) deprecation_logged = any( diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv index ed2aea2d1f..0aadea27af 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv @@ -44,4 +44,4 @@ Kolkata,wikidataId/Q1348 Chennai,wikidataId/Q1352 Mumbai,wikidataId/Q1156 TamilNadu,wikidataId/Q1445 -Andaman & NicobarIslands,wikidataId/Q40888 \ No newline at end of file +Andaman & NicobarIslands,wikidataId/Q40888 diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json index dfed4dc110..fee5f45bb9 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json @@ -3,7 +3,7 @@ "rules": [ { "rule_id": "check_deleted_records_percent", - "description": "Checks that the percentage of deleted records is within the 7% threshold.", + "description": "Checks that the percentage of deleted records is within the 7% threshold. Justification: Accommodates up to 6.94% deletions observed in sub-imports (6.94% in sub_division_wise_annual_rainfall and 5.64% in agriculture) due to: (1) place resolution fixes remapping 'Haryana, Delhi & Chandigarh' from Q1174 to dcid:HaryanaDelhiChandigarh and 'NCT of Delhi' from Q1352 (Chennai) to Q9357528, (2) unit correction for Annual_Amount_FarmInventory_Fruits from Hectare to MetricTon, and (3) official RBI retrospective revisions to provisional historical data.", "validator": "DELETED_RECORDS_PERCENT", "params": { "threshold": 7 @@ -28,4 +28,4 @@ } } ] -} \ No newline at end of file +}