From ea4cdd2362df7ed842426a56eeaa993c07b73a28 Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Tue, 7 Apr 2026 07:21:33 +0000 Subject: [PATCH 1/4] Fixed Code for 3 files --- scripts/un/energy/download.py | 4 ++-- scripts/un/energy/process.py | 3 +++ scripts/un/energy/process_test.py | 1 - 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/un/energy/download.py b/scripts/un/energy/download.py index 4542b4068c..b3dab19d4c 100644 --- a/scripts/un/energy/download.py +++ b/scripts/un/energy/download.py @@ -90,7 +90,6 @@ def download_energy_dataset( str(supported_datasets)) return output_files # Download data in batches of years as the download has a limit of 100k rows. - years_list = list(range(start_year, years_per_batch + 1)) years_list = [str(y) for y in range(start_year, end_year + 1)] batch_years = [ years_list[i:i + years_per_batch] @@ -117,7 +116,8 @@ def download_energy_dataset( if download_successful: logging.info(f"Download of '{download_url}' completed.") for f in os.listdir(output): - output_files.append(os.path.join(output, f)) + if f.endswith('.csv'): + output_files.append(os.path.join(output, f)) else: logging.fatal(f"Download or processing of '{download_url}' failed") return output_files diff --git a/scripts/un/energy/process.py b/scripts/un/energy/process.py index eedbe872d8..5791f7644d 100644 --- a/scripts/un/energy/process.py +++ b/scripts/un/energy/process.py @@ -545,6 +545,9 @@ def process(in_paths: list, with open(mcf_file_path, 'w+', newline='') as f_out_mcf: # Process each CSV input file, one row at a time. for in_file in in_paths: + if not in_file.endswith('.csv'): + logging.info(f'Skipping non-CSV file: {in_file}') + continue logging.info(f'Processing data file: {in_file}') with open(in_file) as csvfile: counters['input_files'] += 1 diff --git a/scripts/un/energy/process_test.py b/scripts/un/energy/process_test.py index 776e45ccbf..bb7fb81c77 100644 --- a/scripts/un/energy/process_test.py +++ b/scripts/un/energy/process_test.py @@ -67,5 +67,4 @@ def test_un_energy_process(self): if __name__ == '__main__': - app.run() unittest.main() From 91d22a72b80b57ddae12b3a9d23be9f72ec9430d Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Wed, 8 Apr 2026 10:59:08 +0000 Subject: [PATCH 2/4] NameError Resolved --- scripts/un/energy/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/un/energy/download.py b/scripts/un/energy/download.py index b3dab19d4c..1ef6e37f13 100644 --- a/scripts/un/energy/download.py +++ b/scripts/un/energy/download.py @@ -83,6 +83,7 @@ def download_energy_dataset( Returns: A list of output files downloaded. """ + output_files = [] supported_datasets = get_all_energy_source_codes() if energy_dataset not in supported_datasets: logging.info( @@ -95,7 +96,6 @@ def download_energy_dataset( years_list[i:i + years_per_batch] for i in range(0, len(years_list), years_per_batch) ] - output_files = [] for year_batch in batch_years: start_year = year_batch[0] end_year = year_batch[-1] From 34050f08451e8cd5b7017ea56d3263409c80aff0 Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Fri, 11 Sep 2026 11:05:57 +0000 Subject: [PATCH 3/4] CensusCountyBusinessPatterns: add retry loop and failure check in shard_input_csv.sh --- .../shard_input_csv.sh | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/scripts/census_county_business_patterns/shard_input_csv.sh b/scripts/census_county_business_patterns/shard_input_csv.sh index 09e0710f23..dc38bf9ec5 100755 --- a/scripts/census_county_business_patterns/shard_input_csv.sh +++ b/scripts/census_county_business_patterns/shard_input_csv.sh @@ -106,19 +106,26 @@ for file in "$SHARD_DIR"/*_shard_*.csv; do echo "INFO: Processing shard: $file (Prefix: $prefix)" >&2 - # Execute the Python processing script in the background (&) - # We assume statvar_processpr.py takes these arguments. - # If there are other arguments (the '...' in your original snippet), add them here. - python3 "$STATVAR_PROCESSOR_SCRIPT" \ - --input_data="$file" \ - --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ - --pv_map="censuscountybusinesspatterns_pvmap.csv" \ - --config_file="censuscountybusinesspatterns_metadata.csv" \ - --output_path="$OUTPUT_FINAL_DIR/output_${prefix}" \ - --counters_print_interval=-1 - # --output_counters="$DEBUG_DIR/counters_${prefix}" \ # uncomment this line to debug the script like to get the details like memory utlization etc. - # Add any other required arguments for statvar_processpr.py here \ - # Run in background + # Execute the Python processing script with retry on failure + success=0 + for attempt in 1 2 3; do + if python3 "$STATVAR_PROCESSOR_SCRIPT" \ + --input_data="$file" \ + --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --pv_map="censuscountybusinesspatterns_pvmap.csv" \ + --config_file="censuscountybusinesspatterns_metadata.csv" \ + --output_path="$OUTPUT_FINAL_DIR/output_${prefix}" \ + --counters_print_interval=-1; then + success=1 + break + fi + echo "WARNING: Processor failed on shard $file (Attempt $attempt/3). Retrying in 15s..." >&2 + sleep 15 + done + if [ $success -ne 1 ]; then + echo "ERROR: Processor failed on shard $file after 3 attempts. Aborting." >&2 + exit 1 + fi # Manage parallelism: pause if too many jobs are running # We monitor the 'statvar_processpr.py' script's processes. From 19a412b593235d5d8b8af20cda35997c0f455d7d Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Mon, 14 Sep 2026 04:28:33 +0000 Subject: [PATCH 4/4] CensusCountyBusinessPatterns: clean up partial artifacts and validate non-empty output --- scripts/census_county_business_patterns/shard_input_csv.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/census_county_business_patterns/shard_input_csv.sh b/scripts/census_county_business_patterns/shard_input_csv.sh index dc38bf9ec5..acfe626c6d 100755 --- a/scripts/census_county_business_patterns/shard_input_csv.sh +++ b/scripts/census_county_business_patterns/shard_input_csv.sh @@ -109,13 +109,14 @@ for file in "$SHARD_DIR"/*_shard_*.csv; do # Execute the Python processing script with retry on failure success=0 for attempt in 1 2 3; do + rm -f "$OUTPUT_FINAL_DIR/output_${prefix}"* if python3 "$STATVAR_PROCESSOR_SCRIPT" \ --input_data="$file" \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ --pv_map="censuscountybusinesspatterns_pvmap.csv" \ --config_file="censuscountybusinesspatterns_metadata.csv" \ --output_path="$OUTPUT_FINAL_DIR/output_${prefix}" \ - --counters_print_interval=-1; then + --counters_print_interval=-1 && [ -s "$OUTPUT_FINAL_DIR/output_${prefix}.csv" ]; then success=1 break fi @@ -123,6 +124,7 @@ for file in "$SHARD_DIR"/*_shard_*.csv; do sleep 15 done if [ $success -ne 1 ]; then + rm -f "$OUTPUT_FINAL_DIR/output_${prefix}"* echo "ERROR: Processor failed on shard $file after 3 attempts. Aborting." >&2 exit 1 fi