From 53cb0ff1e3d9a8811ffa2e0503658661b578de4d Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Mon, 5 Oct 2026 09:45:12 +0000 Subject: [PATCH 01/15] Fix HTTP 429 rate limit errors for USCensusPEP_Annual_Population and USCensusPEP_Sex --- .../pep/annual_population/preprocess.py | 176 ++++++++++--- scripts/us_census/pep/us_pep_sex/process.py | 246 +++++++++++------- 2 files changed, 287 insertions(+), 135 deletions(-) diff --git a/scripts/us_census/pep/annual_population/preprocess.py b/scripts/us_census/pep/annual_population/preprocess.py index a13d5612d6..692c9679e0 100644 --- a/scripts/us_census/pep/annual_population/preprocess.py +++ b/scripts/us_census/pep/annual_population/preprocess.py @@ -43,6 +43,9 @@ import requests import time import re +import random +import shutil +import tempfile from datetime import datetime as dt warnings.filterwarnings('ignore') @@ -64,7 +67,10 @@ def _define_flags(): _MODULE_DIR = os.path.dirname(os.path.abspath(__file__)) _INPUT_FILE_PATH = os.path.join(_MODULE_DIR, 'input_files') +_GCS_FOLDER_PERSISTENT_PATH = os.path.join( + _MODULE_DIR, 'gcs_folder/usa_annual_population_source_files') os.makedirs(_INPUT_FILE_PATH, exist_ok=True) +os.makedirs(_GCS_FOLDER_PERSISTENT_PATH, exist_ok=True) excel_file_name_pattern = r"NST-EST(\d{4})-POP\.xlsx" sys.path.insert(1, _MODULE_DIR) # pylint: disable=wrong-import-position @@ -1075,6 +1081,31 @@ def process(input_path, cleaned_csv_file_path: str, mcf_file_path: str, logging.fatal("No files were successfully processed.") +def probe_url_with_retry(session, url, timeout=10, max_retries=3): + """Probes a URL with HEAD request, handling 429/503 with backoff.""" + for attempt in range(1, max_retries + 1): + try: + response = session.head(url, allow_redirects=True, timeout=timeout) + if response.status_code in (429, 503): + retry_after = response.headers.get("Retry-After") + if retry_after and retry_after.isdigit(): + sleep_time = min(float(retry_after), 30.0) + else: + sleep_time = 2.0 * attempt + random.uniform(0.5, 1.5) + logging.warning( + f"HTTP {response.status_code} probing {url}. Backing off for {sleep_time:.1f}s (attempt {attempt}/{max_retries})." + ) + time.sleep(sleep_time) + continue + return response.status_code == 200 + except requests.exceptions.RequestException as e: + logging.warning( + f"Error probing URL {url}: {e} (attempt {attempt}/{max_retries})" + ) + time.sleep(1.0 * attempt) + return False + + def add_future_year_urls(): """ This method will generate future URLs specified in the urls_to_scan object. If valid URL, it will append to the _FILES_TO_DOWNLOAD global variable. @@ -1091,37 +1122,94 @@ def add_future_year_urls(): "https://www2.census.gov/programs-surveys/popest/tables/2020-{YEAR}/state/totals/NST-EST{YEAR}-POP.xlsx", "https://www2.census.gov/programs-surveys/popest/datasets/2020-{YEAR}/state/totals/NST-EST{YEAR}-POPCHG2020_{YEAR}.csv" ] - # This method will get the latest year URLs available for the years 2023 to 2030 + # Scan from next year down to 2023 (bounded to prevent unnecessary scanning of far future years) + max_scan_year = min(2030, dt.now().year + 1) + session = requests.Session() + session.headers.update({ + 'User-Agent': + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36' + }) for url in urls_to_scan: - for future_year in range(2030, 2022, -1): + for future_year in range(max_scan_year, 2022, -1): url_to_check = url.format(YEAR=future_year) - try: - logging.info(f"checking furute url if available {url_to_check}") - check_url = requests.head(url_to_check, allow_redirects=True) - if check_url.status_code == 200: - logging.info(f"Furute url found {url_to_check}") - _FILES_TO_DOWNLOAD.append({"download_path": url_to_check}) - break - except: - logging.error(f"URL is not accessable {url_to_check}") - - -@retry(tries=5, delay=5, backoff=5) -def download_with_retry(url, file_name_to_save): - logging.info(f"Downloaded file : {file_name_to_save} - URL {url}") - return requests.get(url=url, stream=True) + logging.info(f"checking future url if available {url_to_check}") + if probe_url_with_retry(session, url_to_check): + logging.info(f"Future url found {url_to_check}") + _FILES_TO_DOWNLOAD.append({"download_path": url_to_check}) + break + time.sleep(0.5) + + +def download_single_file_with_retry(session, + url, + dest_input_path, + dest_cache_path, + max_retries=5, + initial_delay=2.0, + backoff_factor=2.0): + """Downloads a single file with exponential backoff and HTTP 429/5xx retry handling.""" + for attempt in range(1, max_retries + 1): + try: + with session.get(url, stream=True, timeout=(15, 120)) as response: + if response.status_code == 200: + with tempfile.NamedTemporaryFile(delete=False) as tmp_file: + for chunk in response.iter_content(chunk_size=65536): + if chunk: + tmp_file.write(chunk) + tmp_file_path = tmp_file.name + + # Save to local input destination + shutil.copy(tmp_file_path, dest_input_path) + # Move to persistent cache destination + shutil.move(tmp_file_path, dest_cache_path) + logging.info(f"Successfully downloaded and cached: {url}") + return True + elif response.status_code in (429, 500, 502, 503, 504): + retry_after = response.headers.get("Retry-After") + if retry_after and retry_after.isdigit(): + sleep_time = min(float(retry_after), 60.0) + else: + sleep_time = initial_delay * (backoff_factor**( + attempt - 1)) + random.uniform(0.5, 1.5) + logging.warning( + f"HTTP {response.status_code} downloading {url}. Attempt {attempt}/{max_retries}. Backing off for {sleep_time:.1f}s." + ) + time.sleep(sleep_time) + else: + logging.error( + f"Failed to download {url} with status code {response.status_code}" + ) + return False + except requests.exceptions.RequestException as e: + sleep_time = initial_delay * (backoff_factor** + (attempt - 1)) + random.uniform( + 0.5, 1.5) + logging.warning( + f"RequestException downloading {url}: {e}. Attempt {attempt}/{max_retries}. Backing off for {sleep_time:.1f}s." + ) + time.sleep(sleep_time) + logging.error(f"Exhausted {max_retries} retries for URL {url}") + return False def download_files(): """ This method will download data from the URLs specified in the input_url.json file. - If any download attempt fails, the script will terminate. + Utilizes persistent caching in gcs_folder to avoid re-downloading unchanged historical files, + and uses polite rate limiting and exponential backoff to avoid HTTP 429 rate limit errors. Args: None Returns: None """ global _FILES_TO_DOWNLOAD - if not os.path.exists(_INPUT_FILE_PATH): - os.makedirs(_INPUT_FILE_PATH) + os.makedirs(_INPUT_FILE_PATH, exist_ok=True) + os.makedirs(_GCS_FOLDER_PERSISTENT_PATH, exist_ok=True) + + session = requests.Session() + session.headers.update({ + 'User-Agent': + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36' + }) + try: for file_to_download in _FILES_TO_DOWNLOAD: file_name_to_save = None @@ -1131,27 +1219,35 @@ def download_files(): file_name_to_save = file_to_download['file_name'] else: file_name_to_save = url.split('/')[-1] - if 'file_path' in file_to_download: - if not os.path.exists( - os.path.join(_INPUT_FILE_PATH, - file_to_download['file_path'])): - os.makedirs( - os.path.join(_INPUT_FILE_PATH, - file_to_download['file_path'])) - file_name_to_save = file_to_download[ - 'file_path'] + file_name_to_save - - response = download_with_retry(url, file_name_to_save) - if response.status_code == 200: - with open(os.path.join(_INPUT_FILE_PATH, file_name_to_save), - 'wb') as f: - f.write(response.content) - file_to_download['is_downloaded'] = True - else: - logging.error( - f"Failed to download {url} with status code {response.status_code}" + + sub_dir = file_to_download.get('file_path', '') + input_dir = os.path.join(_INPUT_FILE_PATH, sub_dir) + cache_dir = os.path.join(_GCS_FOLDER_PERSISTENT_PATH, sub_dir) + os.makedirs(input_dir, exist_ok=True) + os.makedirs(cache_dir, exist_ok=True) + + dest_input_path = os.path.join(input_dir, file_name_to_save) + dest_cache_path = os.path.join(cache_dir, file_name_to_save) + + # Check persistent cache first + if os.path.exists(dest_cache_path) and os.path.getsize( + dest_cache_path) > 0: + logging.info( + f"Skipping download, using cached file: {file_name_to_save}" ) - file_to_download['is_downloaded'] = False + shutil.copy(dest_cache_path, dest_input_path) + file_to_download['is_downloaded'] = True + continue + + # File not cached, download with retry and rate limiting + logging.info(f"Downloading file: {file_name_to_save} - URL {url}") + success = download_single_file_with_retry(session, url, + dest_input_path, + dest_cache_path) + file_to_download['is_downloaded'] = success + # Polite rate limiting between download requests + time.sleep(1.0) + failed_downloads = [ file_to_download['download_path'] for file_to_download in _FILES_TO_DOWNLOAD diff --git a/scripts/us_census/pep/us_pep_sex/process.py b/scripts/us_census/pep/us_pep_sex/process.py index a31429bbbf..8122d87933 100644 --- a/scripts/us_census/pep/us_pep_sex/process.py +++ b/scripts/us_census/pep/us_pep_sex/process.py @@ -18,6 +18,8 @@ import os import sys +import re +import random import pandas as pd import numpy as np from absl import app, flags @@ -1147,6 +1149,28 @@ def is_valid_url(url): return False +def probe_head_url_with_retry(session, url, timeout=5, max_retries=3): + """Probes a URL with HEAD request, handling 429/503 with backoff.""" + for attempt in range(1, max_retries + 1): + try: + response = session.head(url, allow_redirects=True, timeout=timeout) + if response.status_code in (429, 503): + retry_after = response.headers.get("Retry-After") + if retry_after and retry_after.isdigit(): + sleep_time = min(float(retry_after), 30.0) + else: + sleep_time = 2.0 * attempt + random.uniform(0.5, 1.5) + logging.warning( + f"HTTP {response.status_code} probing {url}. Backing off for {sleep_time:.1f}s (attempt {attempt}/{max_retries})." + ) + time.sleep(sleep_time) + continue + return response.status_code == 200 + except requests.exceptions.RequestException as e: + time.sleep(1.0 * attempt) + return False + + def add_future_year_urls(): """ This method adds the future year urls that has to be downloaded @@ -1161,6 +1185,10 @@ def add_future_year_urls(): # Use a requests session for connection pooling to improve efficiency # when making hundreds of HEAD requests. session = requests.Session() + session.headers.update({ + 'User-Agent': + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36' + }) with open(os.path.join(_MODULE_DIR, 'input_url.json'), 'r') as inpit_file: _FILES_TO_DOWNLOAD = json.load(inpit_file) @@ -1196,20 +1224,18 @@ def add_future_year_urls(): # A set to track downloaded URLs for unique {YEAR} and URLs without {i} downloaded_year_urls = set() - # Loop through years in reverse order from 2030 to 2023 - for future_year in range(2030, 2022, -1): # From 2030 to 2023 + max_scan_year = min(2030, dt.now().year + 1) + # Loop through years in reverse order from max_scan_year down to 2023 + for future_year in range(max_scan_year, 2022, -1): - # We check the National CSV first. If it's 404, the whole year is skipped. + # We check the National CSV first. If it's not found, the whole year is skipped. gatekeeper_url = urls_to_scan[0].format(YEAR=future_year) try: - # Use a short 5-second timeout for the check - response = session.head(gatekeeper_url, - allow_redirects=True, - timeout=5) - if response.status_code != 200: + # Use probe with retry for the check + if not probe_head_url_with_retry(session, gatekeeper_url, + timeout=5): logging.info( - f"Skipping year {future_year}: National file not found (status code: {response.status_code})." - ) + f"Skipping year {future_year}: National file not found.") continue except requests.exceptions.RequestException as e: logging.warning( @@ -1226,18 +1252,12 @@ def add_future_year_urls(): url_to_check = url.format(YEAR=YEAR, i=formatted_i) logging.info(f"checking url: {url_to_check}") - try: - # HEAD calls only fetch headers and should complete quickly. - check_url = session.head(url_to_check, - allow_redirects=True, - timeout=5) - if check_url.status_code == 200: - _FILES_TO_DOWNLOAD.append( - {"download_path": url_to_check}) - - except requests.exceptions.RequestException as e: - logging.error( - f"URL is not accessible {url_to_check} due to {e}") + if probe_head_url_with_retry(session, + url_to_check, + timeout=5): + _FILES_TO_DOWNLOAD.append( + {"download_path": url_to_check}) + time.sleep(0.2) else: # This URL does not contain {i}, so we only need to process it once per year url_to_check = url.format(YEAR=YEAR) @@ -1246,55 +1266,112 @@ def add_future_year_urls(): if url_to_check in downloaded_year_urls: continue # Skip this URL if it's already processed - try: - # HEAD calls only fetch headers and should complete quickly. - check_url = session.head(url_to_check, - allow_redirects=True, - timeout=5) - if check_url.status_code == 200: - _FILES_TO_DOWNLOAD.append( - {"download_path": url_to_check}) - downloaded_year_urls.add( - url_to_check) # Mark this URL as processed - - else: - logging.error( - f"URL returned status code {check_url.status_code}: {url_to_check}" - ) - - except requests.exceptions.RequestException as e: - logging.error( - f"URL is not accessible {url_to_check} due to {e}") + if probe_head_url_with_retry(session, url_to_check, timeout=5): + _FILES_TO_DOWNLOAD.append({"download_path": url_to_check}) + downloaded_year_urls.add(url_to_check) + time.sleep(0.2) def cleanup(): - """Delete all old files in the gcs_folder to prevent cache bloat.""" + """Delete all old files in the gcs_folder to prevent cache bloat, + while preserving immutable historical data files. + """ if os.path.exists(_GCS_FOLDER_PERSISTENT_PATH): for file_name in os.listdir(_GCS_FOLDER_PERSISTENT_PATH): file_path = os.path.join(_GCS_FOLDER_PERSISTENT_PATH, file_name) if os.path.isfile(file_path): + # Historical files (pre-2021) should NEVER be evicted from cache + year_matches = re.findall(r'(?:19\d{2}|20[01]\d|2020)', + file_name) + if year_matches: + continue file_age = (time.time() - os.path.getmtime(file_path)) / (24 * 3600) - # Delete ANY file older than the TTL + # Delete recent release files only if older than TTL if file_age > _TTL_DAYS: logging.info(f"Cleaning up old file: {file_name}") os.remove(file_path) -@retry(tries=3, - delay=2, - backoff=2, - exceptions=(requests.RequestException, Exception)) +def download_single_file_with_retry(session, + url, + dest_input_path, + dest_cache_path, + headers, + max_retries=5, + initial_delay=2.0, + backoff_factor=2.0): + """Downloads a single file with exponential backoff and HTTP 429/5xx retry handling.""" + for attempt in range(1, max_retries + 1): + try: + with session.get(url, stream=True, timeout=120, + headers=headers) as response: + content_type = response.headers.get('Content-Type', '') + + # Skip HTML error pages + if 'html' in content_type.lower(): + logging.error( + f"Server returned HTML error page for URL: {url}. Skipping." + ) + return True + + if response.status_code == 200: + with tempfile.NamedTemporaryFile(delete=False) as tmp_file: + for chunk in response.iter_content(chunk_size=65536): + if chunk: + tmp_file.write(chunk) + tmp_file_path = tmp_file.name + + # Copy to local destination + shutil.copy(tmp_file_path, dest_input_path) + # Move to persistent cache destination + shutil.move(tmp_file_path, dest_cache_path) + logging.info(f"Downloaded file: {url}") + return True + elif response.status_code in (429, 500, 502, 503, 504): + retry_after = response.headers.get("Retry-After") + if retry_after and retry_after.isdigit(): + sleep_time = min(float(retry_after), 60.0) + else: + sleep_time = initial_delay * (backoff_factor**( + attempt - 1)) + random.uniform(0.5, 1.5) + logging.warning( + f"HTTP {response.status_code} downloading {url}. Attempt {attempt}/{max_retries}. Backing off for {sleep_time:.1f}s." + ) + time.sleep(sleep_time) + else: + logging.error( + f"Failed to download {url} with status code {response.status_code}" + ) + return False + except requests.exceptions.RequestException as e: + sleep_time = initial_delay * (backoff_factor** + (attempt - 1)) + random.uniform( + 0.5, 1.5) + logging.warning( + f"RequestException downloading {url}: {e}. Attempt {attempt}/{max_retries}. Backing off for {sleep_time:.1f}s." + ) + time.sleep(sleep_time) + logging.error(f"Exhausted {max_retries} retries for URL {url}") + return False + + def download_files(): """ Download files from URLs listed in _FILES_TO_DOWNLOAD. Skips URLs listed in skip_url.json from GCS. + Uses persistent cache in gcs_folder and polite rate limiting with backoff. """ global _FILES_TO_DOWNLOAD session = requests.session() + headers = { + 'User-Agent': + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36' + } - # Ensure the directory exists (it shouldn't "expect" it to be there) + # Ensure the directory exists os.makedirs(_GCS_FOLDER_PERSISTENT_PATH, exist_ok=True) + os.makedirs(_INPUT_FILE_PATH, exist_ok=True) # Get set of already downloaded files downloaded_files = set(os.listdir(_GCS_FOLDER_PERSISTENT_PATH)) @@ -1309,59 +1386,38 @@ def download_files(): else: file_name_to_save = url.split('/')[-1] - # Skip if file already exists in cache (cleanup() already removed stale files) + # Skip if file already exists in cache if file_name_to_save in downloaded_files: file_path = os.path.join(_GCS_FOLDER_PERSISTENT_PATH, file_name_to_save) - logging.info( - f"Skipping download, using cached file: {file_name_to_save}") - - # Make sure to copy the cached file to the input directory! - shutil.copy(file_path, - os.path.join(_INPUT_FILE_PATH, file_name_to_save)) - continue - - headers = {'User-Agent': 'Mozilla/5.0'} - try: - with session.get(url, stream=True, timeout=120, - headers=headers) as response: - response.raise_for_status() - - content_type = response.headers.get('Content-Type', '') - - # Minimal fix: Log error and continue to skip HTML pages - if 'html' in content_type.lower(): - logging.error( - f"Server returned HTML error page for URL: {url}. Skipping." - ) - continue - - if response.status_code == 200: - with tempfile.NamedTemporaryFile(delete=False) as tmp_file: - for chunk in response.iter_content(chunk_size=8192): - if chunk: - tmp_file.write(chunk) - tmp_file_path = tmp_file.name + if os.path.isfile(file_path) and os.path.getsize(file_path) > 0: + logging.info( + f"Skipping download, using cached file: {file_name_to_save}" + ) + shutil.copy(file_path, + os.path.join(_INPUT_FILE_PATH, file_name_to_save)) + file_to_download['is_downloaded'] = True + continue - # Copy to local destination - shutil.copy( - tmp_file_path, - os.path.join(_INPUT_FILE_PATH, file_name_to_save)) + dest_input_path = os.path.join(_INPUT_FILE_PATH, file_name_to_save) + dest_cache_path = os.path.join(_GCS_FOLDER_PERSISTENT_PATH, + file_name_to_save) - # Move to gcs destination (optimized from shutil.copy + os.remove) - shutil.move( - tmp_file_path, - os.path.join(_GCS_FOLDER_PERSISTENT_PATH, - file_name_to_save)) + logging.info(f"Downloading file: {file_name_to_save} - URL {url}") + success = download_single_file_with_retry(session, url, dest_input_path, + dest_cache_path, headers) + file_to_download['is_downloaded'] = success + time.sleep(1.0) - file_to_download['is_downloaded'] = True - logging.info(f"Downloaded file: {url}") - - except Exception as e: - file_to_download['is_downloaded'] = False - logging.error(f"Error downloading {url}: {e}") - raise - time.sleep(1) + failed_downloads = [ + file_to_download['download_path'] + for file_to_download in _FILES_TO_DOWNLOAD + if not file_to_download.get('is_downloaded', False) + ] + if failed_downloads: + raise Exception( + f"Failed to download {len(failed_downloads)} files: {failed_downloads}" + ) return True From 646f21f17bc5b37f1aa2aa23299677d407b9510d Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Mon, 5 Oct 2026 09:57:11 +0000 Subject: [PATCH 02/15] Address review comments: improve historical file cache preservation and clean up temp files in try-finally --- .../pep/annual_population/preprocess.py | 29 ++++++++++----- scripts/us_census/pep/us_pep_sex/process.py | 37 ++++++++++++------- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/scripts/us_census/pep/annual_population/preprocess.py b/scripts/us_census/pep/annual_population/preprocess.py index 692c9679e0..75d5429839 100644 --- a/scripts/us_census/pep/annual_population/preprocess.py +++ b/scripts/us_census/pep/annual_population/preprocess.py @@ -1152,16 +1152,25 @@ def download_single_file_with_retry(session, try: with session.get(url, stream=True, timeout=(15, 120)) as response: if response.status_code == 200: - with tempfile.NamedTemporaryFile(delete=False) as tmp_file: - for chunk in response.iter_content(chunk_size=65536): - if chunk: - tmp_file.write(chunk) - tmp_file_path = tmp_file.name - - # Save to local input destination - shutil.copy(tmp_file_path, dest_input_path) - # Move to persistent cache destination - shutil.move(tmp_file_path, dest_cache_path) + tmp_file_path = None + try: + with tempfile.NamedTemporaryFile( + delete=False) as tmp_file: + tmp_file_path = tmp_file.name + for chunk in response.iter_content( + chunk_size=65536): + if chunk: + tmp_file.write(chunk) + # Save to local input destination + shutil.copy(tmp_file_path, dest_input_path) + # Move to persistent cache destination + shutil.move(tmp_file_path, dest_cache_path) + finally: + if tmp_file_path and os.path.exists(tmp_file_path): + try: + os.remove(tmp_file_path) + except OSError: + pass logging.info(f"Successfully downloaded and cached: {url}") return True elif response.status_code in (429, 500, 502, 503, 504): diff --git a/scripts/us_census/pep/us_pep_sex/process.py b/scripts/us_census/pep/us_pep_sex/process.py index 8122d87933..9667f837a0 100644 --- a/scripts/us_census/pep/us_pep_sex/process.py +++ b/scripts/us_census/pep/us_pep_sex/process.py @@ -1280,10 +1280,10 @@ def cleanup(): for file_name in os.listdir(_GCS_FOLDER_PERSISTENT_PATH): file_path = os.path.join(_GCS_FOLDER_PERSISTENT_PATH, file_name) if os.path.isfile(file_path): - # Historical files (pre-2021) should NEVER be evicted from cache - year_matches = re.findall(r'(?:19\d{2}|20[01]\d|2020)', - file_name) - if year_matches: + # Historical files (pre-2021) should NEVER be evicted from cache. + # We identify recent release files by checking if they contain any 4-digit year >= 2021. + years = [int(yr) for yr in re.findall(r'\d{4}', file_name)] + if not (years and any(yr >= 2021 for yr in years)): continue file_age = (time.time() - os.path.getmtime(file_path)) / (24 * 3600) @@ -1316,16 +1316,25 @@ def download_single_file_with_retry(session, return True if response.status_code == 200: - with tempfile.NamedTemporaryFile(delete=False) as tmp_file: - for chunk in response.iter_content(chunk_size=65536): - if chunk: - tmp_file.write(chunk) - tmp_file_path = tmp_file.name - - # Copy to local destination - shutil.copy(tmp_file_path, dest_input_path) - # Move to persistent cache destination - shutil.move(tmp_file_path, dest_cache_path) + tmp_file_path = None + try: + with tempfile.NamedTemporaryFile( + delete=False) as tmp_file: + tmp_file_path = tmp_file.name + for chunk in response.iter_content( + chunk_size=65536): + if chunk: + tmp_file.write(chunk) + # Copy to local destination + shutil.copy(tmp_file_path, dest_input_path) + # Move to persistent cache destination + shutil.move(tmp_file_path, dest_cache_path) + finally: + if tmp_file_path and os.path.exists(tmp_file_path): + try: + os.remove(tmp_file_path) + except OSError: + pass logging.info(f"Downloaded file: {url}") return True elif response.status_code in (429, 500, 502, 503, 504): From 7f211aa03b0d89e25d7bcd7ca59e75c7c88467df Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Tue, 6 Oct 2026 06:39:38 +0000 Subject: [PATCH 03/15] fixed api issue --- .../pep/annual_population/preprocess.py | 234 +++++------- scripts/us_census/pep/us_pep_sex/process.py | 335 ++++++++---------- 2 files changed, 242 insertions(+), 327 deletions(-) diff --git a/scripts/us_census/pep/annual_population/preprocess.py b/scripts/us_census/pep/annual_population/preprocess.py index 75d5429839..45cb59fe1d 100644 --- a/scripts/us_census/pep/annual_population/preprocess.py +++ b/scripts/us_census/pep/annual_population/preprocess.py @@ -43,9 +43,6 @@ import requests import time import re -import random -import shutil -import tempfile from datetime import datetime as dt warnings.filterwarnings('ignore') @@ -54,6 +51,8 @@ from absl import logging from absl import flags from retry import retry +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry _FLAGS = flags.FLAGS @@ -67,10 +66,7 @@ def _define_flags(): _MODULE_DIR = os.path.dirname(os.path.abspath(__file__)) _INPUT_FILE_PATH = os.path.join(_MODULE_DIR, 'input_files') -_GCS_FOLDER_PERSISTENT_PATH = os.path.join( - _MODULE_DIR, 'gcs_folder/usa_annual_population_source_files') os.makedirs(_INPUT_FILE_PATH, exist_ok=True) -os.makedirs(_GCS_FOLDER_PERSISTENT_PATH, exist_ok=True) excel_file_name_pattern = r"NST-EST(\d{4})-POP\.xlsx" sys.path.insert(1, _MODULE_DIR) # pylint: disable=wrong-import-position @@ -1081,29 +1077,21 @@ def process(input_path, cleaned_csv_file_path: str, mcf_file_path: str, logging.fatal("No files were successfully processed.") -def probe_url_with_retry(session, url, timeout=10, max_retries=3): - """Probes a URL with HEAD request, handling 429/503 with backoff.""" - for attempt in range(1, max_retries + 1): - try: - response = session.head(url, allow_redirects=True, timeout=timeout) - if response.status_code in (429, 503): - retry_after = response.headers.get("Retry-After") - if retry_after and retry_after.isdigit(): - sleep_time = min(float(retry_after), 30.0) - else: - sleep_time = 2.0 * attempt + random.uniform(0.5, 1.5) - logging.warning( - f"HTTP {response.status_code} probing {url}. Backing off for {sleep_time:.1f}s (attempt {attempt}/{max_retries})." - ) - time.sleep(sleep_time) - continue - return response.status_code == 200 - except requests.exceptions.RequestException as e: - logging.warning( - f"Error probing URL {url}: {e} (attempt {attempt}/{max_retries})" - ) - time.sleep(1.0 * attempt) - return False +def _create_retry_session() -> requests.Session: + """Creates a requests Session configured with retry and backoff for transient errors.""" + session = requests.Session() + session.headers.update({'User-Agent': 'Mozilla/5.0'}) + retries = Retry( + total=5, + backoff_factor=2, + status_forcelist=[429, 500, 502, 503, 504, 520], + allowed_methods=["HEAD", "GET"], + respect_retry_after_header=True, + ) + adapter = HTTPAdapter(max_retries=retries) + session.mount("https://", adapter) + session.mount("http://", adapter) + return session def add_future_year_urls(): @@ -1117,108 +1105,69 @@ def add_future_year_urls(): with open(os.path.join(_MODULE_DIR, 'input_url.json'), 'r') as inpit_file: _FILES_TO_DOWNLOAD = json.load(inpit_file) urls_to_scan = [ - "https://www2.census.gov/programs-surveys/popest/datasets/2020-{YEAR}/cities/totals/sub-est{YEAR}.csv", - "https://www2.census.gov/programs-surveys/popest/datasets/2020-{YEAR}/counties/totals/co-est{YEAR}-alldata.csv", - "https://www2.census.gov/programs-surveys/popest/tables/2020-{YEAR}/state/totals/NST-EST{YEAR}-POP.xlsx", - "https://www2.census.gov/programs-surveys/popest/datasets/2020-{YEAR}/state/totals/NST-EST{YEAR}-POPCHG2020_{YEAR}.csv" + ("https://www2.census.gov/programs-surveys/popest/datasets/2020-{YEAR}/cities/totals/sub-est{YEAR}.csv", + 2025), + ("https://www2.census.gov/programs-surveys/popest/datasets/2020-{YEAR}/counties/totals/co-est{YEAR}-alldata.csv", + 2025), + ("https://www2.census.gov/programs-surveys/popest/tables/2020-{YEAR}/state/totals/NST-EST{YEAR}-POP.xlsx", + 2025), + ("https://www2.census.gov/programs-surveys/popest/datasets/2020-{YEAR}/state/totals/NST-EST{YEAR}-POPCHG2020_{YEAR}.csv", + 2023), ] - # Scan from next year down to 2023 (bounded to prevent unnecessary scanning of far future years) - max_scan_year = min(2030, dt.now().year + 1) - session = requests.Session() - session.headers.update({ - 'User-Agent': - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36' - }) - for url in urls_to_scan: - for future_year in range(max_scan_year, 2022, -1): + session = _create_retry_session() + # This method will get the latest year URLs available for the years 2023 to 2030 + for url, baseline_year in urls_to_scan: + found = False + for future_year in range(2030, 2022, -1): url_to_check = url.format(YEAR=future_year) - logging.info(f"checking future url if available {url_to_check}") - if probe_url_with_retry(session, url_to_check): - logging.info(f"Future url found {url_to_check}") - _FILES_TO_DOWNLOAD.append({"download_path": url_to_check}) - break - time.sleep(0.5) - - -def download_single_file_with_retry(session, - url, - dest_input_path, - dest_cache_path, - max_retries=5, - initial_delay=2.0, - backoff_factor=2.0): - """Downloads a single file with exponential backoff and HTTP 429/5xx retry handling.""" - for attempt in range(1, max_retries + 1): - try: - with session.get(url, stream=True, timeout=(15, 120)) as response: - if response.status_code == 200: - tmp_file_path = None - try: - with tempfile.NamedTemporaryFile( - delete=False) as tmp_file: - tmp_file_path = tmp_file.name - for chunk in response.iter_content( - chunk_size=65536): - if chunk: - tmp_file.write(chunk) - # Save to local input destination - shutil.copy(tmp_file_path, dest_input_path) - # Move to persistent cache destination - shutil.move(tmp_file_path, dest_cache_path) - finally: - if tmp_file_path and os.path.exists(tmp_file_path): - try: - os.remove(tmp_file_path) - except OSError: - pass - logging.info(f"Successfully downloaded and cached: {url}") - return True - elif response.status_code in (429, 500, 502, 503, 504): - retry_after = response.headers.get("Retry-After") - if retry_after and retry_after.isdigit(): - sleep_time = min(float(retry_after), 60.0) - else: - sleep_time = initial_delay * (backoff_factor**( - attempt - 1)) + random.uniform(0.5, 1.5) - logging.warning( - f"HTTP {response.status_code} downloading {url}. Attempt {attempt}/{max_retries}. Backing off for {sleep_time:.1f}s." - ) - time.sleep(sleep_time) - else: - logging.error( - f"Failed to download {url} with status code {response.status_code}" - ) - return False - except requests.exceptions.RequestException as e: - sleep_time = initial_delay * (backoff_factor** - (attempt - 1)) + random.uniform( - 0.5, 1.5) + try: + logging.info(f"checking furute url if available {url_to_check}") + check_url = session.head(url_to_check, + allow_redirects=True, + timeout=30) + time.sleep(0.2) + content_type = check_url.headers.get('Content-Type', '') + if check_url.status_code == 200 and 'html' not in content_type.lower( + ): + logging.info(f"Furute url found {url_to_check}") + _FILES_TO_DOWNLOAD.append({"download_path": url_to_check}) + found = True + break + except requests.RequestException as e: + logging.error(f"URL is not accessable {url_to_check}: {e}") + if not found: + fallback_url = url.format(YEAR=baseline_year) logging.warning( - f"RequestException downloading {url}: {e}. Attempt {attempt}/{max_retries}. Backing off for {sleep_time:.1f}s." + f"No future URL confirmed via HEAD probe for {url}; falling back to known baseline URL: {fallback_url}" ) - time.sleep(sleep_time) - logging.error(f"Exhausted {max_retries} retries for URL {url}") - return False + _FILES_TO_DOWNLOAD.append({"download_path": fallback_url}) + + +@retry(tries=5, + delay=5, + backoff=2, + exceptions=(requests.RequestException, ValueError)) +def download_with_retry(url, file_name_to_save): + logging.info(f"Downloaded file : {file_name_to_save} - URL {url}") + headers = {'User-Agent': 'Mozilla/5.0'} + response = requests.get(url=url, stream=True, timeout=60, headers=headers) + response.raise_for_status() + content_type = response.headers.get('Content-Type', '') + if 'html' in content_type.lower(): + raise ValueError(f"Server returned HTML error page for URL: {url}") + return response def download_files(): """ This method will download data from the URLs specified in the input_url.json file. - Utilizes persistent caching in gcs_folder to avoid re-downloading unchanged historical files, - and uses polite rate limiting and exponential backoff to avoid HTTP 429 rate limit errors. + If any download attempt fails, the script will terminate. Args: None Returns: None """ global _FILES_TO_DOWNLOAD - os.makedirs(_INPUT_FILE_PATH, exist_ok=True) - os.makedirs(_GCS_FOLDER_PERSISTENT_PATH, exist_ok=True) - - session = requests.Session() - session.headers.update({ - 'User-Agent': - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36' - }) - + if not os.path.exists(_INPUT_FILE_PATH): + os.makedirs(_INPUT_FILE_PATH) try: for file_to_download in _FILES_TO_DOWNLOAD: file_name_to_save = None @@ -1228,35 +1177,28 @@ def download_files(): file_name_to_save = file_to_download['file_name'] else: file_name_to_save = url.split('/')[-1] - - sub_dir = file_to_download.get('file_path', '') - input_dir = os.path.join(_INPUT_FILE_PATH, sub_dir) - cache_dir = os.path.join(_GCS_FOLDER_PERSISTENT_PATH, sub_dir) - os.makedirs(input_dir, exist_ok=True) - os.makedirs(cache_dir, exist_ok=True) - - dest_input_path = os.path.join(input_dir, file_name_to_save) - dest_cache_path = os.path.join(cache_dir, file_name_to_save) - - # Check persistent cache first - if os.path.exists(dest_cache_path) and os.path.getsize( - dest_cache_path) > 0: - logging.info( - f"Skipping download, using cached file: {file_name_to_save}" - ) - shutil.copy(dest_cache_path, dest_input_path) + if 'file_path' in file_to_download: + if not os.path.exists( + os.path.join(_INPUT_FILE_PATH, + file_to_download['file_path'])): + os.makedirs( + os.path.join(_INPUT_FILE_PATH, + file_to_download['file_path'])) + file_name_to_save = file_to_download[ + 'file_path'] + file_name_to_save + + response = download_with_retry(url, file_name_to_save) + if response.status_code == 200: + with open(os.path.join(_INPUT_FILE_PATH, file_name_to_save), + 'wb') as f: + f.write(response.content) file_to_download['is_downloaded'] = True - continue - - # File not cached, download with retry and rate limiting - logging.info(f"Downloading file: {file_name_to_save} - URL {url}") - success = download_single_file_with_retry(session, url, - dest_input_path, - dest_cache_path) - file_to_download['is_downloaded'] = success - # Polite rate limiting between download requests - time.sleep(1.0) - + else: + logging.error( + f"Failed to download {url} with status code {response.status_code}" + ) + file_to_download['is_downloaded'] = False + time.sleep(0.2) failed_downloads = [ file_to_download['download_path'] for file_to_download in _FILES_TO_DOWNLOAD diff --git a/scripts/us_census/pep/us_pep_sex/process.py b/scripts/us_census/pep/us_pep_sex/process.py index 9667f837a0..cc55750516 100644 --- a/scripts/us_census/pep/us_pep_sex/process.py +++ b/scripts/us_census/pep/us_pep_sex/process.py @@ -18,8 +18,6 @@ import os import sys -import re -import random import pandas as pd import numpy as np from absl import app, flags @@ -31,6 +29,8 @@ from absl import logging from absl import flags from retry import retry +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry from google.cloud import storage # GCS Client import tempfile @@ -1132,13 +1132,31 @@ def fetch_skip_urls_from_gcs(GCS_BUCKET_NAME: str, return skip_urls +def _create_retry_session() -> requests.Session: + """Creates a requests Session configured with retry and backoff for transient errors.""" + session = requests.Session() + session.headers.update({'User-Agent': 'Mozilla/5.0'}) + retries = Retry( + total=5, + backoff_factor=2, + status_forcelist=[429, 500, 502, 503, 504, 520], + allowed_methods=["HEAD", "GET"], + respect_retry_after_header=True, + ) + adapter = HTTPAdapter(max_retries=retries) + session.mount("https://", adapter) + session.mount("http://", adapter) + return session + + def is_valid_url(url): try: - response = requests.get(url, timeout=20) + session = _create_retry_session() + response = session.get(url, timeout=20) if response.status_code != 200: return False content_type = response.headers.get("Content-Type", "") - if "text/html" in content_type: + if "text/html" in content_type.lower(): # Might be an error page disguised as 200 if b"error" in response.content.lower() or len( response.content) < 500: @@ -1149,28 +1167,6 @@ def is_valid_url(url): return False -def probe_head_url_with_retry(session, url, timeout=5, max_retries=3): - """Probes a URL with HEAD request, handling 429/503 with backoff.""" - for attempt in range(1, max_retries + 1): - try: - response = session.head(url, allow_redirects=True, timeout=timeout) - if response.status_code in (429, 503): - retry_after = response.headers.get("Retry-After") - if retry_after and retry_after.isdigit(): - sleep_time = min(float(retry_after), 30.0) - else: - sleep_time = 2.0 * attempt + random.uniform(0.5, 1.5) - logging.warning( - f"HTTP {response.status_code} probing {url}. Backing off for {sleep_time:.1f}s (attempt {attempt}/{max_retries})." - ) - time.sleep(sleep_time) - continue - return response.status_code == 200 - except requests.exceptions.RequestException as e: - time.sleep(1.0 * attempt) - return False - - def add_future_year_urls(): """ This method adds the future year urls that has to be downloaded @@ -1182,13 +1178,8 @@ def add_future_year_urls(): # Initialize the list to store files to download _FILES_TO_DOWNLOAD = [] - # Use a requests session for connection pooling to improve efficiency - # when making hundreds of HEAD requests. - session = requests.Session() - session.headers.update({ - 'User-Agent': - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36' - }) + # Use a retry-enabled requests session for connection pooling and 429/5xx backoff + session = _create_retry_session() with open(os.path.join(_MODULE_DIR, 'input_url.json'), 'r') as inpit_file: _FILES_TO_DOWNLOAD = json.load(inpit_file) @@ -1224,18 +1215,20 @@ def add_future_year_urls(): # A set to track downloaded URLs for unique {YEAR} and URLs without {i} downloaded_year_urls = set() - max_scan_year = min(2030, dt.now().year + 1) - # Loop through years in reverse order from max_scan_year down to 2023 - for future_year in range(max_scan_year, 2022, -1): + # Loop through years in reverse order from 2030 to 2023 + for future_year in range(2030, 2022, -1): # From 2030 to 2023 - # We check the National CSV first. If it's not found, the whole year is skipped. + # We check the National CSV first. If it's 404, the whole year is skipped. gatekeeper_url = urls_to_scan[0].format(YEAR=future_year) try: - # Use probe with retry for the check - if not probe_head_url_with_retry(session, gatekeeper_url, - timeout=5): + response = session.head(gatekeeper_url, + allow_redirects=True, + timeout=15) + time.sleep(0.2) + if response.status_code != 200: logging.info( - f"Skipping year {future_year}: National file not found.") + f"Skipping year {future_year}: National file not found (status code: {response.status_code})." + ) continue except requests.exceptions.RequestException as e: logging.warning( @@ -1252,12 +1245,19 @@ def add_future_year_urls(): url_to_check = url.format(YEAR=YEAR, i=formatted_i) logging.info(f"checking url: {url_to_check}") - if probe_head_url_with_retry(session, - url_to_check, - timeout=5): - _FILES_TO_DOWNLOAD.append( - {"download_path": url_to_check}) - time.sleep(0.2) + try: + # HEAD calls only fetch headers and should complete quickly. + check_url = session.head(url_to_check, + allow_redirects=True, + timeout=15) + time.sleep(0.1) + if check_url.status_code == 200: + _FILES_TO_DOWNLOAD.append( + {"download_path": url_to_check}) + + except requests.exceptions.RequestException as e: + logging.error( + f"URL is not accessible {url_to_check} due to {e}") else: # This URL does not contain {i}, so we only need to process it once per year url_to_check = url.format(YEAR=YEAR) @@ -1266,119 +1266,86 @@ def add_future_year_urls(): if url_to_check in downloaded_year_urls: continue # Skip this URL if it's already processed - if probe_head_url_with_retry(session, url_to_check, timeout=5): - _FILES_TO_DOWNLOAD.append({"download_path": url_to_check}) - downloaded_year_urls.add(url_to_check) - time.sleep(0.2) - + try: + # HEAD calls only fetch headers and should complete quickly. + check_url = session.head(url_to_check, + allow_redirects=True, + timeout=15) + time.sleep(0.1) + if check_url.status_code == 200: + _FILES_TO_DOWNLOAD.append( + {"download_path": url_to_check}) + downloaded_year_urls.add( + url_to_check) # Mark this URL as processed -def cleanup(): - """Delete all old files in the gcs_folder to prevent cache bloat, - while preserving immutable historical data files. - """ - if os.path.exists(_GCS_FOLDER_PERSISTENT_PATH): - for file_name in os.listdir(_GCS_FOLDER_PERSISTENT_PATH): - file_path = os.path.join(_GCS_FOLDER_PERSISTENT_PATH, file_name) - if os.path.isfile(file_path): - # Historical files (pre-2021) should NEVER be evicted from cache. - # We identify recent release files by checking if they contain any 4-digit year >= 2021. - years = [int(yr) for yr in re.findall(r'\d{4}', file_name)] - if not (years and any(yr >= 2021 for yr in years)): - continue - file_age = (time.time() - os.path.getmtime(file_path)) / (24 * - 3600) - # Delete recent release files only if older than TTL - if file_age > _TTL_DAYS: - logging.info(f"Cleaning up old file: {file_name}") - os.remove(file_path) - - -def download_single_file_with_retry(session, - url, - dest_input_path, - dest_cache_path, - headers, - max_retries=5, - initial_delay=2.0, - backoff_factor=2.0): - """Downloads a single file with exponential backoff and HTTP 429/5xx retry handling.""" - for attempt in range(1, max_retries + 1): - try: - with session.get(url, stream=True, timeout=120, - headers=headers) as response: - content_type = response.headers.get('Content-Type', '') + else: + logging.error( + f"URL returned status code {check_url.status_code}: {url_to_check}" + ) - # Skip HTML error pages - if 'html' in content_type.lower(): + except requests.exceptions.RequestException as e: logging.error( - f"Server returned HTML error page for URL: {url}. Skipping." - ) - return True + f"URL is not accessible {url_to_check} due to {e}") - if response.status_code == 200: - tmp_file_path = None - try: - with tempfile.NamedTemporaryFile( - delete=False) as tmp_file: - tmp_file_path = tmp_file.name - for chunk in response.iter_content( - chunk_size=65536): - if chunk: - tmp_file.write(chunk) - # Copy to local destination - shutil.copy(tmp_file_path, dest_input_path) - # Move to persistent cache destination - shutil.move(tmp_file_path, dest_cache_path) - finally: - if tmp_file_path and os.path.exists(tmp_file_path): - try: - os.remove(tmp_file_path) - except OSError: - pass - logging.info(f"Downloaded file: {url}") - return True - elif response.status_code in (429, 500, 502, 503, 504): - retry_after = response.headers.get("Retry-After") - if retry_after and retry_after.isdigit(): - sleep_time = min(float(retry_after), 60.0) - else: - sleep_time = initial_delay * (backoff_factor**( - attempt - 1)) + random.uniform(0.5, 1.5) - logging.warning( - f"HTTP {response.status_code} downloading {url}. Attempt {attempt}/{max_retries}. Backing off for {sleep_time:.1f}s." - ) - time.sleep(sleep_time) - else: - logging.error( - f"Failed to download {url} with status code {response.status_code}" - ) - return False - except requests.exceptions.RequestException as e: - sleep_time = initial_delay * (backoff_factor** - (attempt - 1)) + random.uniform( - 0.5, 1.5) - logging.warning( - f"RequestException downloading {url}: {e}. Attempt {attempt}/{max_retries}. Backing off for {sleep_time:.1f}s." - ) - time.sleep(sleep_time) - logging.error(f"Exhausted {max_retries} retries for URL {url}") - return False + +def _get_target_file_name(file_to_download: dict) -> str: + """Returns the local filename to save for a download entry.""" + if 'file_name' in file_to_download and len( + file_to_download['file_name']) > 5: + return file_to_download['file_name'] + return file_to_download['download_path'].split('/')[-1] +def cleanup(): + """Delete unreferenced or expired files in gcs_folder after downloads complete.""" + if not os.path.exists(_GCS_FOLDER_PERSISTENT_PATH): + return + active_files = set() + if _FILES_TO_DOWNLOAD: + active_files = { + _get_target_file_name(item) for item in _FILES_TO_DOWNLOAD + } + for file_name in os.listdir(_GCS_FOLDER_PERSISTENT_PATH): + file_path = os.path.join(_GCS_FOLDER_PERSISTENT_PATH, file_name) + if os.path.isfile(file_path): + file_age = (time.time() - os.path.getmtime(file_path)) / (24 * 3600) + if file_age > _TTL_DAYS and file_name not in active_files: + logging.info(f"Cleaning up old file: {file_name}") + os.remove(file_path) + + +@retry(tries=5, + delay=3, + backoff=2, + exceptions=(requests.RequestException, ValueError)) +def _download_single_file(session: requests.Session, url: str) -> str: + """Downloads a single URL to a temporary file, retrying on HTTP or HTML error responses.""" + with session.get(url, stream=True, timeout=120) as response: + response.raise_for_status() + content_type = response.headers.get('Content-Type', '') + if 'html' in content_type.lower(): + raise ValueError(f"Server returned HTML error page for URL: {url}") + + with tempfile.NamedTemporaryFile(delete=False) as tmp_file: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + tmp_file.write(chunk) + return tmp_file.name + + +@retry(tries=3, + delay=2, + backoff=2, + exceptions=(requests.RequestException, Exception)) def download_files(): """ Download files from URLs listed in _FILES_TO_DOWNLOAD. Skips URLs listed in skip_url.json from GCS. - Uses persistent cache in gcs_folder and polite rate limiting with backoff. """ global _FILES_TO_DOWNLOAD - session = requests.session() - headers = { - 'User-Agent': - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36' - } + session = _create_retry_session() - # Ensure the directory exists + # Ensure the directory exists (it shouldn't "expect" it to be there) os.makedirs(_GCS_FOLDER_PERSISTENT_PATH, exist_ok=True) os.makedirs(_INPUT_FILE_PATH, exist_ok=True) @@ -1386,47 +1353,55 @@ def download_files(): downloaded_files = set(os.listdir(_GCS_FOLDER_PERSISTENT_PATH)) for file_to_download in _FILES_TO_DOWNLOAD: - file_name_to_save = None url = file_to_download['download_path'] - - if 'file_name' in file_to_download and len( - file_to_download['file_name']) > 5: - file_name_to_save = file_to_download['file_name'] - else: - file_name_to_save = url.split('/')[-1] - - # Skip if file already exists in cache - if file_name_to_save in downloaded_files: - file_path = os.path.join(_GCS_FOLDER_PERSISTENT_PATH, - file_name_to_save) - if os.path.isfile(file_path) and os.path.getsize(file_path) > 0: + file_name_to_save = _get_target_file_name(file_to_download) + cached_file_path = os.path.join(_GCS_FOLDER_PERSISTENT_PATH, + file_name_to_save) + + # Reuse cached file if it exists and is within TTL + if file_name_to_save in downloaded_files and os.path.isfile( + cached_file_path): + file_age = (time.time() - os.path.getmtime(cached_file_path)) / ( + 24 * 3600) + if file_age <= _TTL_DAYS: logging.info( f"Skipping download, using cached file: {file_name_to_save}" ) - shutil.copy(file_path, + shutil.copy(cached_file_path, os.path.join(_INPUT_FILE_PATH, file_name_to_save)) file_to_download['is_downloaded'] = True continue + logging.info( + f"Cached file {file_name_to_save} expired (age: {file_age:.1f} days); refreshing." + ) - dest_input_path = os.path.join(_INPUT_FILE_PATH, file_name_to_save) - dest_cache_path = os.path.join(_GCS_FOLDER_PERSISTENT_PATH, - file_name_to_save) + try: + tmp_file_path = _download_single_file(session, url) - logging.info(f"Downloading file: {file_name_to_save} - URL {url}") - success = download_single_file_with_retry(session, url, dest_input_path, - dest_cache_path, headers) - file_to_download['is_downloaded'] = success - time.sleep(1.0) + # Copy to local destination + shutil.copy(tmp_file_path, + os.path.join(_INPUT_FILE_PATH, file_name_to_save)) - failed_downloads = [ - file_to_download['download_path'] - for file_to_download in _FILES_TO_DOWNLOAD - if not file_to_download.get('is_downloaded', False) - ] - if failed_downloads: - raise Exception( - f"Failed to download {len(failed_downloads)} files: {failed_downloads}" - ) + # Move to gcs destination (optimized from shutil.copy + os.remove) + shutil.move(tmp_file_path, cached_file_path) + downloaded_files.add(file_name_to_save) + + file_to_download['is_downloaded'] = True + logging.info(f"Downloaded file: {url}") + + except Exception as e: + if os.path.isfile(cached_file_path): + logging.warning( + f"Failed to refresh {url} after retries ({e}); falling back to cached file: {file_name_to_save}" + ) + shutil.copy(cached_file_path, + os.path.join(_INPUT_FILE_PATH, file_name_to_save)) + file_to_download['is_downloaded'] = True + continue + file_to_download['is_downloaded'] = False + logging.error(f"Error downloading {url}: {e}") + raise + time.sleep(0.5) return True @@ -1456,9 +1431,6 @@ def main(_): mcf_path = data_file_path + os.sep + mcf_name tmcf_path = data_file_path + os.sep + tmcf_name - # Perform cleanup of old files - cleanup() - download_status = True if mode == "" or mode == "download": # Get the config path from the flags @@ -1471,6 +1443,7 @@ def main(_): # download & process add_future_year_urls() download_status = download_files() + cleanup() if download_status and (mode == "" or mode == "process"): try: loader = PopulationEstimateBySex(_INPUT_FILE_PATH, cleaned_csv_path, From 464afa4ab83382428f97fbd6ac7bc508d947bc31 Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Tue, 6 Oct 2026 12:06:04 +0000 Subject: [PATCH 04/15] fixed api issue --- .../pep/annual_population/preprocess.py | 86 +++++++++--------- scripts/us_census/pep/us_pep_sex/process.py | 91 +++++++++++-------- 2 files changed, 100 insertions(+), 77 deletions(-) diff --git a/scripts/us_census/pep/annual_population/preprocess.py b/scripts/us_census/pep/annual_population/preprocess.py index 45cb59fe1d..dd270fa113 100644 --- a/scripts/us_census/pep/annual_population/preprocess.py +++ b/scripts/us_census/pep/annual_population/preprocess.py @@ -366,8 +366,8 @@ def _process_nationals_1990_1999(ip_file: str) -> pd.DataFrame: "Count_Person_Female" ] data_df.columns = df_cols - data_df = data_df[(data_df["Age"] == "All Age") & - (data_df["Year"].str.startswith("July"))].reset_index( + data_df = data_df[(data_df["Age"] == "All Age") + & (data_df["Year"].str.startswith("July"))].reset_index( drop=True) data_df["Year"] = data_df["Year"].str.replace("July 1, ", "") data_df = data_df.drop( @@ -495,8 +495,7 @@ def _process_nationals_2029(file_path: str) -> pd.DataFrame: unpivot_cols = ["2022", "2023"] # extend df_cols & unpivot_cols with all years > 2023 newly_added_years = [ - str(x) - for x in data_df.columns.to_list() + str(x) for x in data_df.columns.to_list() if isinstance(x, int) and x > 2023 ] df_cols.extend(newly_added_years) @@ -550,8 +549,7 @@ def _process_states_2029(file_path: str) -> pd.DataFrame: header=0).iloc[2:3].values.flatten().tolist() if year_column_row: newly_added_years = [ - str(int(x)) - for x in year_column_row + str(int(x)) for x in year_column_row if isinstance(x, float) and x > 2023 ] df_cols.extend(newly_added_years) @@ -852,7 +850,8 @@ def _process_city_1990_1999(file_path: str) -> pd.DataFrame: data[1] = "06000" loc = "geoId/" + f"{int(data[0]):02d}" \ + f"{int(data[1]):05d}" - for year, val in dict(zip(cols, data[-len(cols):])).items(): + for year, val in dict(zip(cols, + data[-len(cols):])).items(): outfile.write(f"{year},{loc},{val}\n") data_df = pd.read_csv("out.csv", header=0) os.remove("out.csv") @@ -1038,7 +1037,7 @@ def process(input_path, cleaned_csv_file_path: str, mcf_file_path: str, county_df = _process_counties(file) data_df = pd.concat([nat_df, county_df]) elif file_name in ["e7079co.txt", "99c8_00.txt" - ] or "co-est" in file_name: + ] or "co-est" in file_name: data_df = _process_counties(file) elif file_name in [ 'su-99-7_us.txt', "sub-est2010-alt.csv", @@ -1102,8 +1101,8 @@ def add_future_year_urls(): Returns: None """ global _FILES_TO_DOWNLOAD - with open(os.path.join(_MODULE_DIR, 'input_url.json'), 'r') as inpit_file: - _FILES_TO_DOWNLOAD = json.load(inpit_file) + with open(os.path.join(_MODULE_DIR, 'input_url.json'), 'r') as input_file: + _FILES_TO_DOWNLOAD = json.load(input_file) urls_to_scan = [ ("https://www2.census.gov/programs-surveys/popest/datasets/2020-{YEAR}/cities/totals/sub-est{YEAR}.csv", 2025), @@ -1121,7 +1120,8 @@ def add_future_year_urls(): for future_year in range(2030, 2022, -1): url_to_check = url.format(YEAR=future_year) try: - logging.info(f"checking furute url if available {url_to_check}") + logging.info( + f"checking future url if available {url_to_check}") check_url = session.head(url_to_check, allow_redirects=True, timeout=30) @@ -1129,12 +1129,12 @@ def add_future_year_urls(): content_type = check_url.headers.get('Content-Type', '') if check_url.status_code == 200 and 'html' not in content_type.lower( ): - logging.info(f"Furute url found {url_to_check}") + logging.info(f"Future url found {url_to_check}") _FILES_TO_DOWNLOAD.append({"download_path": url_to_check}) found = True break except requests.RequestException as e: - logging.error(f"URL is not accessable {url_to_check}: {e}") + logging.error(f"URL is not accessible {url_to_check}: {e}") if not found: fallback_url = url.format(YEAR=baseline_year) logging.warning( @@ -1147,15 +1147,18 @@ def add_future_year_urls(): delay=5, backoff=2, exceptions=(requests.RequestException, ValueError)) -def download_with_retry(url, file_name_to_save): - logging.info(f"Downloaded file : {file_name_to_save} - URL {url}") - headers = {'User-Agent': 'Mozilla/5.0'} - response = requests.get(url=url, stream=True, timeout=60, headers=headers) - response.raise_for_status() - content_type = response.headers.get('Content-Type', '') - if 'html' in content_type.lower(): - raise ValueError(f"Server returned HTML error page for URL: {url}") - return response +def download_with_retry(session: requests.Session, url: str, dest_path: str): + logging.info(f"Downloading file from URL: {url} -> {dest_path}") + with session.get(url, stream=True, timeout=120) as response: + response.raise_for_status() + content_type = response.headers.get('Content-Type', '') + if 'html' in content_type.lower(): + raise ValueError(f"Server returned HTML error page for URL: {url}") + with open(dest_path, 'wb') as f: + for chunk in response.iter_content(chunk_size=65536): + if chunk: + f.write(chunk) + logging.info(f"Downloaded file: {dest_path}") def download_files(): @@ -1168,6 +1171,7 @@ def download_files(): global _FILES_TO_DOWNLOAD if not os.path.exists(_INPUT_FILE_PATH): os.makedirs(_INPUT_FILE_PATH) + session = _create_retry_session() try: for file_to_download in _FILES_TO_DOWNLOAD: file_name_to_save = None @@ -1178,26 +1182,25 @@ def download_files(): else: file_name_to_save = url.split('/')[-1] if 'file_path' in file_to_download: - if not os.path.exists( - os.path.join(_INPUT_FILE_PATH, - file_to_download['file_path'])): - os.makedirs( - os.path.join(_INPUT_FILE_PATH, - file_to_download['file_path'])) - file_name_to_save = file_to_download[ - 'file_path'] + file_name_to_save - - response = download_with_retry(url, file_name_to_save) - if response.status_code == 200: - with open(os.path.join(_INPUT_FILE_PATH, file_name_to_save), - 'wb') as f: - f.write(response.content) + target_dir = os.path.join(_INPUT_FILE_PATH, + file_to_download['file_path']) + if not os.path.exists(target_dir): + os.makedirs(target_dir, exist_ok=True) + file_name_to_save = os.path.join(file_to_download['file_path'], + file_name_to_save) + + dest_path = os.path.join(_INPUT_FILE_PATH, file_name_to_save) + try: + download_with_retry(session, url, dest_path) file_to_download['is_downloaded'] = True - else: - logging.error( - f"Failed to download {url} with status code {response.status_code}" - ) + except Exception as e: + logging.error(f"Failed to download {url}: {e}") file_to_download['is_downloaded'] = False + if os.path.exists(dest_path): + try: + os.remove(dest_path) + except OSError: + pass time.sleep(0.2) failed_downloads = [ file_to_download['download_path'] @@ -1220,7 +1223,8 @@ def main(_): # Defining Output file names data_file_path = os.path.join(_MODULE_DIR, OUTPUT_DIR) os.makedirs(data_file_path, exist_ok=True) - cleaned_csv_path = os.path.join(data_file_path, "usa_annual_population.csv") + cleaned_csv_path = os.path.join(data_file_path, + "usa_annual_population.csv") mcf_path = os.path.join(data_file_path, "usa_annual_population.mcf") tmcf_path = os.path.join(data_file_path, "usa_annual_population.tmcf") diff --git a/scripts/us_census/pep/us_pep_sex/process.py b/scripts/us_census/pep/us_pep_sex/process.py index cc55750516..ba6da5f5cf 100644 --- a/scripts/us_census/pep/us_pep_sex/process.py +++ b/scripts/us_census/pep/us_pep_sex/process.py @@ -209,8 +209,8 @@ def _national_1990_2000(file_path: str) -> pd.DataFrame: 'Year', 'Age', 'Total', 'Count_Person_Male', 'Count_Person_Female' ] # total age is required as we are bring age in a seperate import - df = df[(df["Age"] == "All Age") & - (df["Year"].str.startswith("July"))].reset_index(drop=True) + df = df[(df["Age"] == "All Age") + & (df["Year"].str.startswith("July"))].reset_index(drop=True) df["Year"] = df["Year"].str.replace("July 1, ", "") # dropping unwanted columns df = df.drop(columns=df.columns.difference( @@ -560,8 +560,8 @@ def _state_2010_2020(file_path: str) -> pd.DataFrame: df = df.drop(columns=df.columns.difference( ['Year', 'Count_Person_Male', 'Count_Person_Female', 'geo_ID'])) - df = df[(df['Year'] != 'April2010Census') & - (df['Year'] != 'April2010Estimate') & + df = df[(df['Year'] != 'April2010Census') + & (df['Year'] != 'April2010Estimate') & (df['Year'] != 'April2020') & (df['Year'] != '2020')] df['Measurement_Method'] = 'dcAggregate/CensusPEPSurvey_PartialAggregate' return df @@ -882,8 +882,8 @@ def _county_2010_2020(file_path: str) -> pd.DataFrame: df = df.drop(columns=df.columns.difference( ['Year', 'Count_Person_Male', 'Count_Person_Female', 'geo_ID'])) - df = df[(df['Year'] != 'April2010Census') & - (df['Year'] != 'April2010Estimate') & + df = df[(df['Year'] != 'April2010Census') + & (df['Year'] != 'April2010Estimate') & (df['Year'] != 'April2020')] df['Measurement_Method'] = 'CensusPEPSurvey' return df @@ -930,8 +930,8 @@ def _county_latest(file_path: str) -> pd.DataFrame: df = df.drop(columns=df.columns.difference( ['Year', 'Count_Person_Male', 'Count_Person_Female', 'geo_ID'])) - df = df[(df['Year'] != 'April2020Estimate') & - (df['Year'] != 'July2020')] + df = df[(df['Year'] != 'April2020Estimate') + & (df['Year'] != 'July2020')] df['Measurement_Method'] = 'CensusPEPSurvey' return df @@ -1149,19 +1149,18 @@ def _create_retry_session() -> requests.Session: return session -def is_valid_url(url): +def is_valid_url(session: requests.Session, url: str) -> bool: try: - session = _create_retry_session() - response = session.get(url, timeout=20) - if response.status_code != 200: - return False - content_type = response.headers.get("Content-Type", "") - if "text/html" in content_type.lower(): - # Might be an error page disguised as 200 - if b"error" in response.content.lower() or len( - response.content) < 500: + with session.get(url, stream=True, timeout=20) as response: + if response.status_code != 200: return False - return True + content_type = response.headers.get("Content-Type", "") + if "text/html" in content_type.lower(): + # Might be an error page disguised as 200 + chunk = next(response.iter_content(chunk_size=1024), b"") + if b"error" in chunk.lower() or len(chunk) < 500: + return False + return True except Exception as e: logging.warning(f"Error checking URL: {url} - {e}") return False @@ -1180,8 +1179,8 @@ def add_future_year_urls(): # Use a retry-enabled requests session for connection pooling and 429/5xx backoff session = _create_retry_session() - with open(os.path.join(_MODULE_DIR, 'input_url.json'), 'r') as inpit_file: - _FILES_TO_DOWNLOAD = json.load(inpit_file) + with open(os.path.join(_MODULE_DIR, 'input_url.json'), 'r') as input_file: + _FILES_TO_DOWNLOAD = json.load(input_file) GCS_BUCKET_NAME, GCS_SKIP_FILE_PATH = extract_gcs_info() # Read URLs from the GCS-hosted config file @@ -1190,19 +1189,19 @@ def add_future_year_urls(): skip_urls = fetch_skip_urls_from_gcs(GCS_BUCKET_NAME, GCS_SKIP_FILE_PATH) logging.info(f"Fetched {len(skip_urls)} URLs to skip from GCS.") - logging.info(f"Urls fetched from the json to skip:{skip_urls}") + logging.info(f"Urls fetched from the json to skip: {skip_urls}") #Filter based on skip list + live URL check _FILES_TO_DOWNLOAD = [ url for url in _FILES_TO_DOWNLOAD if (url["download_path"] not in skip_urls # not suspicious, keep it or (url["download_path"] in skip_urls and is_valid_url( - url["download_path"])) # suspicious, but passes check - ) + session, url["download_path"])) # suspicious, but passes check + ) ] logging.info( - f"Historical urls Fetched from input_json:{_FILES_TO_DOWNLOAD}") + f"Historical urls fetched from input_json: {_FILES_TO_DOWNLOAD}") # List of URLs with placeholders for {YEAR} and {i} urls_to_scan = [ @@ -1303,12 +1302,14 @@ def cleanup(): active_files = set() if _FILES_TO_DOWNLOAD: active_files = { - _get_target_file_name(item) for item in _FILES_TO_DOWNLOAD + _get_target_file_name(item) + for item in _FILES_TO_DOWNLOAD } for file_name in os.listdir(_GCS_FOLDER_PERSISTENT_PATH): file_path = os.path.join(_GCS_FOLDER_PERSISTENT_PATH, file_name) if os.path.isfile(file_path): - file_age = (time.time() - os.path.getmtime(file_path)) / (24 * 3600) + file_age = (time.time() - os.path.getmtime(file_path)) / (24 * + 3600) if file_age > _TTL_DAYS and file_name not in active_files: logging.info(f"Cleaning up old file: {file_name}") os.remove(file_path) @@ -1326,11 +1327,21 @@ def _download_single_file(session: requests.Session, url: str) -> str: if 'html' in content_type.lower(): raise ValueError(f"Server returned HTML error page for URL: {url}") - with tempfile.NamedTemporaryFile(delete=False) as tmp_file: - for chunk in response.iter_content(chunk_size=8192): - if chunk: - tmp_file.write(chunk) - return tmp_file.name + tmp_file_path = None + try: + with tempfile.NamedTemporaryFile(delete=False) as tmp_file: + tmp_file_path = tmp_file.name + for chunk in response.iter_content(chunk_size=8192): + if chunk: + tmp_file.write(chunk) + return tmp_file_path + except Exception: + if tmp_file_path and os.path.exists(tmp_file_path): + try: + os.remove(tmp_file_path) + except OSError: + pass + raise @retry(tries=3, @@ -1361,8 +1372,8 @@ def download_files(): # Reuse cached file if it exists and is within TTL if file_name_to_save in downloaded_files and os.path.isfile( cached_file_path): - file_age = (time.time() - os.path.getmtime(cached_file_path)) / ( - 24 * 3600) + file_age = (time.time() - + os.path.getmtime(cached_file_path)) / (24 * 3600) if file_age <= _TTL_DAYS: logging.info( f"Skipping download, using cached file: {file_name_to_save}" @@ -1375,6 +1386,7 @@ def download_files(): f"Cached file {file_name_to_save} expired (age: {file_age:.1f} days); refreshing." ) + tmp_file_path = None try: tmp_file_path = _download_single_file(session, url) @@ -1384,12 +1396,18 @@ def download_files(): # Move to gcs destination (optimized from shutil.copy + os.remove) shutil.move(tmp_file_path, cached_file_path) + tmp_file_path = None downloaded_files.add(file_name_to_save) file_to_download['is_downloaded'] = True logging.info(f"Downloaded file: {url}") except Exception as e: + if tmp_file_path and os.path.exists(tmp_file_path): + try: + os.remove(tmp_file_path) + except OSError: + pass if os.path.isfile(cached_file_path): logging.warning( f"Failed to refresh {url} after retries ({e}); falling back to cached file: {file_name_to_save}" @@ -1446,8 +1464,9 @@ def main(_): cleanup() if download_status and (mode == "" or mode == "process"): try: - loader = PopulationEstimateBySex(_INPUT_FILE_PATH, cleaned_csv_path, - mcf_path, tmcf_path) + loader = PopulationEstimateBySex(_INPUT_FILE_PATH, + cleaned_csv_path, mcf_path, + tmcf_path) loader.process() # The persistent folder is intentionally kept to allow for TTL caching. From 934f345e5f7af5c2c890699b9e4077de35d2751e Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Tue, 6 Oct 2026 13:39:50 +0000 Subject: [PATCH 05/15] fixed api issue --- .../pep/annual_population/preprocess.py | 21 +++++++------- scripts/us_census/pep/us_pep_sex/process.py | 29 +++++++++---------- 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/scripts/us_census/pep/annual_population/preprocess.py b/scripts/us_census/pep/annual_population/preprocess.py index dd270fa113..bc91c22bc2 100644 --- a/scripts/us_census/pep/annual_population/preprocess.py +++ b/scripts/us_census/pep/annual_population/preprocess.py @@ -366,8 +366,8 @@ def _process_nationals_1990_1999(ip_file: str) -> pd.DataFrame: "Count_Person_Female" ] data_df.columns = df_cols - data_df = data_df[(data_df["Age"] == "All Age") - & (data_df["Year"].str.startswith("July"))].reset_index( + data_df = data_df[(data_df["Age"] == "All Age") & + (data_df["Year"].str.startswith("July"))].reset_index( drop=True) data_df["Year"] = data_df["Year"].str.replace("July 1, ", "") data_df = data_df.drop( @@ -495,7 +495,8 @@ def _process_nationals_2029(file_path: str) -> pd.DataFrame: unpivot_cols = ["2022", "2023"] # extend df_cols & unpivot_cols with all years > 2023 newly_added_years = [ - str(x) for x in data_df.columns.to_list() + str(x) + for x in data_df.columns.to_list() if isinstance(x, int) and x > 2023 ] df_cols.extend(newly_added_years) @@ -549,7 +550,8 @@ def _process_states_2029(file_path: str) -> pd.DataFrame: header=0).iloc[2:3].values.flatten().tolist() if year_column_row: newly_added_years = [ - str(int(x)) for x in year_column_row + str(int(x)) + for x in year_column_row if isinstance(x, float) and x > 2023 ] df_cols.extend(newly_added_years) @@ -850,8 +852,7 @@ def _process_city_1990_1999(file_path: str) -> pd.DataFrame: data[1] = "06000" loc = "geoId/" + f"{int(data[0]):02d}" \ + f"{int(data[1]):05d}" - for year, val in dict(zip(cols, - data[-len(cols):])).items(): + for year, val in dict(zip(cols, data[-len(cols):])).items(): outfile.write(f"{year},{loc},{val}\n") data_df = pd.read_csv("out.csv", header=0) os.remove("out.csv") @@ -1037,7 +1038,7 @@ def process(input_path, cleaned_csv_file_path: str, mcf_file_path: str, county_df = _process_counties(file) data_df = pd.concat([nat_df, county_df]) elif file_name in ["e7079co.txt", "99c8_00.txt" - ] or "co-est" in file_name: + ] or "co-est" in file_name: data_df = _process_counties(file) elif file_name in [ 'su-99-7_us.txt', "sub-est2010-alt.csv", @@ -1120,8 +1121,7 @@ def add_future_year_urls(): for future_year in range(2030, 2022, -1): url_to_check = url.format(YEAR=future_year) try: - logging.info( - f"checking future url if available {url_to_check}") + logging.info(f"checking future url if available {url_to_check}") check_url = session.head(url_to_check, allow_redirects=True, timeout=30) @@ -1223,8 +1223,7 @@ def main(_): # Defining Output file names data_file_path = os.path.join(_MODULE_DIR, OUTPUT_DIR) os.makedirs(data_file_path, exist_ok=True) - cleaned_csv_path = os.path.join(data_file_path, - "usa_annual_population.csv") + cleaned_csv_path = os.path.join(data_file_path, "usa_annual_population.csv") mcf_path = os.path.join(data_file_path, "usa_annual_population.mcf") tmcf_path = os.path.join(data_file_path, "usa_annual_population.tmcf") diff --git a/scripts/us_census/pep/us_pep_sex/process.py b/scripts/us_census/pep/us_pep_sex/process.py index ba6da5f5cf..c5c6c6c8e1 100644 --- a/scripts/us_census/pep/us_pep_sex/process.py +++ b/scripts/us_census/pep/us_pep_sex/process.py @@ -209,8 +209,8 @@ def _national_1990_2000(file_path: str) -> pd.DataFrame: 'Year', 'Age', 'Total', 'Count_Person_Male', 'Count_Person_Female' ] # total age is required as we are bring age in a seperate import - df = df[(df["Age"] == "All Age") - & (df["Year"].str.startswith("July"))].reset_index(drop=True) + df = df[(df["Age"] == "All Age") & + (df["Year"].str.startswith("July"))].reset_index(drop=True) df["Year"] = df["Year"].str.replace("July 1, ", "") # dropping unwanted columns df = df.drop(columns=df.columns.difference( @@ -560,8 +560,8 @@ def _state_2010_2020(file_path: str) -> pd.DataFrame: df = df.drop(columns=df.columns.difference( ['Year', 'Count_Person_Male', 'Count_Person_Female', 'geo_ID'])) - df = df[(df['Year'] != 'April2010Census') - & (df['Year'] != 'April2010Estimate') & + df = df[(df['Year'] != 'April2010Census') & + (df['Year'] != 'April2010Estimate') & (df['Year'] != 'April2020') & (df['Year'] != '2020')] df['Measurement_Method'] = 'dcAggregate/CensusPEPSurvey_PartialAggregate' return df @@ -882,8 +882,8 @@ def _county_2010_2020(file_path: str) -> pd.DataFrame: df = df.drop(columns=df.columns.difference( ['Year', 'Count_Person_Male', 'Count_Person_Female', 'geo_ID'])) - df = df[(df['Year'] != 'April2010Census') - & (df['Year'] != 'April2010Estimate') & + df = df[(df['Year'] != 'April2010Census') & + (df['Year'] != 'April2010Estimate') & (df['Year'] != 'April2020')] df['Measurement_Method'] = 'CensusPEPSurvey' return df @@ -930,8 +930,8 @@ def _county_latest(file_path: str) -> pd.DataFrame: df = df.drop(columns=df.columns.difference( ['Year', 'Count_Person_Male', 'Count_Person_Female', 'geo_ID'])) - df = df[(df['Year'] != 'April2020Estimate') - & (df['Year'] != 'July2020')] + df = df[(df['Year'] != 'April2020Estimate') & + (df['Year'] != 'July2020')] df['Measurement_Method'] = 'CensusPEPSurvey' return df @@ -1197,7 +1197,7 @@ def add_future_year_urls(): if (url["download_path"] not in skip_urls # not suspicious, keep it or (url["download_path"] in skip_urls and is_valid_url( session, url["download_path"])) # suspicious, but passes check - ) + ) ] logging.info( @@ -1302,14 +1302,12 @@ def cleanup(): active_files = set() if _FILES_TO_DOWNLOAD: active_files = { - _get_target_file_name(item) - for item in _FILES_TO_DOWNLOAD + _get_target_file_name(item) for item in _FILES_TO_DOWNLOAD } for file_name in os.listdir(_GCS_FOLDER_PERSISTENT_PATH): file_path = os.path.join(_GCS_FOLDER_PERSISTENT_PATH, file_name) if os.path.isfile(file_path): - file_age = (time.time() - os.path.getmtime(file_path)) / (24 * - 3600) + file_age = (time.time() - os.path.getmtime(file_path)) / (24 * 3600) if file_age > _TTL_DAYS and file_name not in active_files: logging.info(f"Cleaning up old file: {file_name}") os.remove(file_path) @@ -1464,9 +1462,8 @@ def main(_): cleanup() if download_status and (mode == "" or mode == "process"): try: - loader = PopulationEstimateBySex(_INPUT_FILE_PATH, - cleaned_csv_path, mcf_path, - tmcf_path) + loader = PopulationEstimateBySex(_INPUT_FILE_PATH, cleaned_csv_path, + mcf_path, tmcf_path) loader.process() # The persistent folder is intentionally kept to allow for TTL caching. From 99dfa2e51abfa01011646cfbc8b2094c044ef02f Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Tue, 6 Oct 2026 18:36:44 +0000 Subject: [PATCH 06/15] fixed api issue --- .../pep/annual_population/preprocess.py | 34 ++++---- .../annual_population/validation_config.json | 9 ++ scripts/us_census/pep/us_pep_sex/process.py | 87 ++++++++++--------- 3 files changed, 73 insertions(+), 57 deletions(-) diff --git a/scripts/us_census/pep/annual_population/preprocess.py b/scripts/us_census/pep/annual_population/preprocess.py index bc91c22bc2..641ab8c9ad 100644 --- a/scripts/us_census/pep/annual_population/preprocess.py +++ b/scripts/us_census/pep/annual_population/preprocess.py @@ -366,8 +366,8 @@ def _process_nationals_1990_1999(ip_file: str) -> pd.DataFrame: "Count_Person_Female" ] data_df.columns = df_cols - data_df = data_df[(data_df["Age"] == "All Age") & - (data_df["Year"].str.startswith("July"))].reset_index( + data_df = data_df[(data_df["Age"] == "All Age") + & (data_df["Year"].str.startswith("July"))].reset_index( drop=True) data_df["Year"] = data_df["Year"].str.replace("July 1, ", "") data_df = data_df.drop( @@ -495,9 +495,8 @@ def _process_nationals_2029(file_path: str) -> pd.DataFrame: unpivot_cols = ["2022", "2023"] # extend df_cols & unpivot_cols with all years > 2023 newly_added_years = [ - str(x) - for x in data_df.columns.to_list() - if isinstance(x, int) and x > 2023 + str(int(x)) for x in data_df.columns + if isinstance(x, (int, float)) and not pd.isna(x) and int(x) > 2023 ] df_cols.extend(newly_added_years) unpivot_cols.extend(newly_added_years) @@ -550,15 +549,15 @@ def _process_states_2029(file_path: str) -> pd.DataFrame: header=0).iloc[2:3].values.flatten().tolist() if year_column_row: newly_added_years = [ - str(int(x)) - for x in year_column_row - if isinstance(x, float) and x > 2023 + str(int(x)) for x in year_column_row + if isinstance(x, (int, + float)) and not pd.isna(x) and int(x) > 2023 ] df_cols.extend(newly_added_years) unpivot_cols.extend(newly_added_years) - except: - # this error can be ignored - pass + except (ValueError, IndexError, KeyError, TypeError) as e: + logging.warning( + f"Could not extract dynamic year columns from {file_path}: {e}") data_df = _load_data_df(path=file_path, file_format="xlsx", header=8) data_df.columns = df_cols @@ -852,7 +851,8 @@ def _process_city_1990_1999(file_path: str) -> pd.DataFrame: data[1] = "06000" loc = "geoId/" + f"{int(data[0]):02d}" \ + f"{int(data[1]):05d}" - for year, val in dict(zip(cols, data[-len(cols):])).items(): + for year, val in dict(zip(cols, + data[-len(cols):])).items(): outfile.write(f"{year},{loc},{val}\n") data_df = pd.read_csv("out.csv", header=0) os.remove("out.csv") @@ -1001,7 +1001,7 @@ def process(input_path, cleaned_csv_file_path: str, mcf_file_path: str, _process_nationals_2010_2020(file, op_file) data_df = _load_data_df(op_file, "csv", 0) os.remove(op_file) - elif "NST-EST2023-POPCHG2020_2023.csv" in file: + elif re.search(r"NST-EST\d{4}-POPCHG2020_\d{4}\.csv", file_name): _process_nationals_2020_2029(file, op_file) data_df = _load_data_df(op_file, "csv", 0) os.remove(op_file) @@ -1038,7 +1038,7 @@ def process(input_path, cleaned_csv_file_path: str, mcf_file_path: str, county_df = _process_counties(file) data_df = pd.concat([nat_df, county_df]) elif file_name in ["e7079co.txt", "99c8_00.txt" - ] or "co-est" in file_name: + ] or "co-est" in file_name: data_df = _process_counties(file) elif file_name in [ 'su-99-7_us.txt', "sub-est2010-alt.csv", @@ -1121,7 +1121,8 @@ def add_future_year_urls(): for future_year in range(2030, 2022, -1): url_to_check = url.format(YEAR=future_year) try: - logging.info(f"checking future url if available {url_to_check}") + logging.info( + f"checking future url if available {url_to_check}") check_url = session.head(url_to_check, allow_redirects=True, timeout=30) @@ -1223,7 +1224,8 @@ def main(_): # Defining Output file names data_file_path = os.path.join(_MODULE_DIR, OUTPUT_DIR) os.makedirs(data_file_path, exist_ok=True) - cleaned_csv_path = os.path.join(data_file_path, "usa_annual_population.csv") + cleaned_csv_path = os.path.join(data_file_path, + "usa_annual_population.csv") mcf_path = os.path.join(data_file_path, "usa_annual_population.mcf") tmcf_path = os.path.join(data_file_path, "usa_annual_population.tmcf") diff --git a/scripts/us_census/pep/annual_population/validation_config.json b/scripts/us_census/pep/annual_population/validation_config.json index 4068a1c3f9..23af91eab4 100644 --- a/scripts/us_census/pep/annual_population/validation_config.json +++ b/scripts/us_census/pep/annual_population/validation_config.json @@ -24,6 +24,15 @@ "golden_files": "../../../../golden_data/golden_observations.csv", "input_files": "../../../../output_files/usa_annual_population.csv" } + }, + { + "rule_id": "check_date_freshness", + "description": "Verifies that all StatVars meet freshness requirements (within 2 years of current year).", + "validator": "SQL_VALIDATOR", + "params": { + "query": "SELECT MIN(MaxDate) AS latest_date FROM stats", + "condition": "CAST(SUBSTR(latest_date, 1, 4) AS INT) >= YEAR(CURRENT_DATE) - 2" + } } ] } \ No newline at end of file diff --git a/scripts/us_census/pep/us_pep_sex/process.py b/scripts/us_census/pep/us_pep_sex/process.py index c5c6c6c8e1..1e01d75cbd 100644 --- a/scripts/us_census/pep/us_pep_sex/process.py +++ b/scripts/us_census/pep/us_pep_sex/process.py @@ -209,8 +209,8 @@ def _national_1990_2000(file_path: str) -> pd.DataFrame: 'Year', 'Age', 'Total', 'Count_Person_Male', 'Count_Person_Female' ] # total age is required as we are bring age in a seperate import - df = df[(df["Age"] == "All Age") & - (df["Year"].str.startswith("July"))].reset_index(drop=True) + df = df[(df["Age"] == "All Age") + & (df["Year"].str.startswith("July"))].reset_index(drop=True) df["Year"] = df["Year"].str.replace("July 1, ", "") # dropping unwanted columns df = df.drop(columns=df.columns.difference( @@ -560,8 +560,8 @@ def _state_2010_2020(file_path: str) -> pd.DataFrame: df = df.drop(columns=df.columns.difference( ['Year', 'Count_Person_Male', 'Count_Person_Female', 'geo_ID'])) - df = df[(df['Year'] != 'April2010Census') & - (df['Year'] != 'April2010Estimate') & + df = df[(df['Year'] != 'April2010Census') + & (df['Year'] != 'April2010Estimate') & (df['Year'] != 'April2020') & (df['Year'] != '2020')] df['Measurement_Method'] = 'dcAggregate/CensusPEPSurvey_PartialAggregate' return df @@ -882,8 +882,8 @@ def _county_2010_2020(file_path: str) -> pd.DataFrame: df = df.drop(columns=df.columns.difference( ['Year', 'Count_Person_Male', 'Count_Person_Female', 'geo_ID'])) - df = df[(df['Year'] != 'April2010Census') & - (df['Year'] != 'April2010Estimate') & + df = df[(df['Year'] != 'April2010Census') + & (df['Year'] != 'April2010Estimate') & (df['Year'] != 'April2020')] df['Measurement_Method'] = 'CensusPEPSurvey' return df @@ -930,8 +930,8 @@ def _county_latest(file_path: str) -> pd.DataFrame: df = df.drop(columns=df.columns.difference( ['Year', 'Count_Person_Male', 'Count_Person_Female', 'geo_ID'])) - df = df[(df['Year'] != 'April2020Estimate') & - (df['Year'] != 'July2020')] + df = df[(df['Year'] != 'April2020Estimate') + & (df['Year'] != 'July2020')] df['Measurement_Method'] = 'CensusPEPSurvey' return df @@ -1054,33 +1054,32 @@ def process(self): # Iterate from 2023 to 2029 and add mappings for years dynamically for file_year in range(2023, 2030): # For the 'national', 'state', and 'county' entries, use the same method for years 2023 to 2029 - if file_year < dt.now().year: - file_to_function_mapping[ - f"nc-est{file_year}-agesex-"] = _national_latest - file_to_function_mapping[ - f"nc-est{file_year}-agesex-r"] = _national_latest - - file_to_function_mapping[ - f"sc-est{file_year}-syasex-2"] = _state_latest - file_to_function_mapping[ - f"sc-est{file_year}-syasex-3"] = _state_latest - file_to_function_mapping[ - f"sc-est{file_year}-syasex-4"] = _state_latest - file_to_function_mapping[ - f"sc-est{file_year}-syasex-5"] = _state_latest - file_to_function_mapping[ - f"sc-est{file_year}-syasex-0"] = _state_latest - file_to_function_mapping[ - f"sc-est{file_year}-syasex-1"] = _state_latest - file_to_function_mapping[ - f"sc-est{file_year}-syasex-"] = _state_latest - file_to_function_mapping[ - f"sc-est{file_year}-agesex-"] = _state_latest - - file_to_function_mapping[ - f"cc-est{file_year}-agesex-"] = _county_latest - file_to_function_mapping[ - f"cc-est{file_year}-agesex-a"] = _county_latest + file_to_function_mapping[ + f"nc-est{file_year}-agesex-"] = _national_latest + file_to_function_mapping[ + f"nc-est{file_year}-agesex-r"] = _national_latest + + file_to_function_mapping[ + f"sc-est{file_year}-syasex-2"] = _state_latest + file_to_function_mapping[ + f"sc-est{file_year}-syasex-3"] = _state_latest + file_to_function_mapping[ + f"sc-est{file_year}-syasex-4"] = _state_latest + file_to_function_mapping[ + f"sc-est{file_year}-syasex-5"] = _state_latest + file_to_function_mapping[ + f"sc-est{file_year}-syasex-0"] = _state_latest + file_to_function_mapping[ + f"sc-est{file_year}-syasex-1"] = _state_latest + file_to_function_mapping[ + f"sc-est{file_year}-syasex-"] = _state_latest + file_to_function_mapping[ + f"sc-est{file_year}-agesex-"] = _state_latest + + file_to_function_mapping[ + f"cc-est{file_year}-agesex-"] = _county_latest + file_to_function_mapping[ + f"cc-est{file_year}-agesex-a"] = _county_latest df = file_to_function_mapping[file_name](file_path) @@ -1094,7 +1093,7 @@ def process(self): logging.info(f"No of files processed {processed_count}") # Log the resulting dictionary logging.info(f"File-to-Function mappings: {file_to_function_mapping}") - if processed_count == total_files_to_process & total_files_to_process > 0: + if processed_count == total_files_to_process and total_files_to_process > 0: final_df = _states_full_to_short_form(final_df, 'geo_ID', "geo_ID") final_df = _add_geo_id(final_df, "geo_ID", "geo_ID") final_df = pd.melt( @@ -1197,7 +1196,7 @@ def add_future_year_urls(): if (url["download_path"] not in skip_urls # not suspicious, keep it or (url["download_path"] in skip_urls and is_valid_url( session, url["download_path"])) # suspicious, but passes check - ) + ) ] logging.info( @@ -1286,6 +1285,9 @@ def add_future_year_urls(): logging.error( f"URL is not accessible {url_to_check} due to {e}") + # Terminate year scan once latest confirmed vintage is processed + break + def _get_target_file_name(file_to_download: dict) -> str: """Returns the local filename to save for a download entry.""" @@ -1302,12 +1304,14 @@ def cleanup(): active_files = set() if _FILES_TO_DOWNLOAD: active_files = { - _get_target_file_name(item) for item in _FILES_TO_DOWNLOAD + _get_target_file_name(item) + for item in _FILES_TO_DOWNLOAD } for file_name in os.listdir(_GCS_FOLDER_PERSISTENT_PATH): file_path = os.path.join(_GCS_FOLDER_PERSISTENT_PATH, file_name) if os.path.isfile(file_path): - file_age = (time.time() - os.path.getmtime(file_path)) / (24 * 3600) + file_age = (time.time() - os.path.getmtime(file_path)) / (24 * + 3600) if file_age > _TTL_DAYS and file_name not in active_files: logging.info(f"Cleaning up old file: {file_name}") os.remove(file_path) @@ -1462,8 +1466,9 @@ def main(_): cleanup() if download_status and (mode == "" or mode == "process"): try: - loader = PopulationEstimateBySex(_INPUT_FILE_PATH, cleaned_csv_path, - mcf_path, tmcf_path) + loader = PopulationEstimateBySex(_INPUT_FILE_PATH, + cleaned_csv_path, mcf_path, + tmcf_path) loader.process() # The persistent folder is intentionally kept to allow for TTL caching. From 3cf2da17073a8711173f36c3c5a031d1f01af853 Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Tue, 6 Oct 2026 18:49:00 +0000 Subject: [PATCH 07/15] fixed api issue --- scripts/us_census/pep/us_pep_sex/validation_config.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/us_census/pep/us_pep_sex/validation_config.json b/scripts/us_census/pep/us_pep_sex/validation_config.json index c4bde5edcc..876227ff87 100644 --- a/scripts/us_census/pep/us_pep_sex/validation_config.json +++ b/scripts/us_census/pep/us_pep_sex/validation_config.json @@ -24,6 +24,15 @@ "golden_files": "../../../../golden_data/golden_observations.csv", "input_files": "../../../../output/population_estimate_sex.csv" } + }, + { + "rule_id": "check_date_freshness", + "description": "Verifies that all StatVars meet freshness requirements (within 2 years of current year).", + "validator": "SQL_VALIDATOR", + "params": { + "query": "SELECT MIN(MaxDate) AS latest_date FROM stats", + "condition": "CAST(SUBSTR(latest_date, 1, 4) AS INT) >= YEAR(CURRENT_DATE) - 2" + } } ] } From 9f2fac4fcae60de2025d10813d619512c9b9822b Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Wed, 7 Oct 2026 05:53:27 +0000 Subject: [PATCH 08/15] fixed api issue --- .../us_census/pep/annual_population/manifest.json | 3 ++- .../pep/annual_population/validation_config.json | 2 +- scripts/us_census/pep/us_pep_sex/process.py | 14 ++++++++++---- .../pep/us_pep_sex/validation_config.json | 2 +- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/scripts/us_census/pep/annual_population/manifest.json b/scripts/us_census/pep/annual_population/manifest.json index 7655d3c516..4346cb61a2 100644 --- a/scripts/us_census/pep/annual_population/manifest.json +++ b/scripts/us_census/pep/annual_population/manifest.json @@ -12,7 +12,8 @@ "preprocess.py --mode=process" ], "source_files": [ - "input_files/*" + "input_files/*", + "golden_data/*.csv" ], "import_inputs": [ { diff --git a/scripts/us_census/pep/annual_population/validation_config.json b/scripts/us_census/pep/annual_population/validation_config.json index 23af91eab4..4f448c54d7 100644 --- a/scripts/us_census/pep/annual_population/validation_config.json +++ b/scripts/us_census/pep/annual_population/validation_config.json @@ -31,7 +31,7 @@ "validator": "SQL_VALIDATOR", "params": { "query": "SELECT MIN(MaxDate) AS latest_date FROM stats", - "condition": "CAST(SUBSTR(latest_date, 1, 4) AS INT) >= YEAR(CURRENT_DATE) - 2" + "condition": "latest_date IS NOT NULL AND CAST(SUBSTR(latest_date, 1, 4) AS INT) >= YEAR(CURRENT_DATE) - 2" } } ] diff --git a/scripts/us_census/pep/us_pep_sex/process.py b/scripts/us_census/pep/us_pep_sex/process.py index 1e01d75cbd..27b7a0e418 100644 --- a/scripts/us_census/pep/us_pep_sex/process.py +++ b/scripts/us_census/pep/us_pep_sex/process.py @@ -1223,9 +1223,10 @@ def add_future_year_urls(): allow_redirects=True, timeout=15) time.sleep(0.2) - if response.status_code != 200: + content_type = response.headers.get('Content-Type', '') + if response.status_code != 200 or 'html' in content_type.lower(): logging.info( - f"Skipping year {future_year}: National file not found (status code: {response.status_code})." + f"Skipping year {future_year}: National file not found or is HTML (status code: {response.status_code})." ) continue except requests.exceptions.RequestException as e: @@ -1249,7 +1250,10 @@ def add_future_year_urls(): allow_redirects=True, timeout=15) time.sleep(0.1) - if check_url.status_code == 200: + content_type = check_url.headers.get( + 'Content-Type', '') + if check_url.status_code == 200 and 'html' not in content_type.lower( + ): _FILES_TO_DOWNLOAD.append( {"download_path": url_to_check}) @@ -1270,7 +1274,9 @@ def add_future_year_urls(): allow_redirects=True, timeout=15) time.sleep(0.1) - if check_url.status_code == 200: + content_type = check_url.headers.get('Content-Type', '') + if check_url.status_code == 200 and 'html' not in content_type.lower( + ): _FILES_TO_DOWNLOAD.append( {"download_path": url_to_check}) downloaded_year_urls.add( diff --git a/scripts/us_census/pep/us_pep_sex/validation_config.json b/scripts/us_census/pep/us_pep_sex/validation_config.json index 876227ff87..586e23600f 100644 --- a/scripts/us_census/pep/us_pep_sex/validation_config.json +++ b/scripts/us_census/pep/us_pep_sex/validation_config.json @@ -31,7 +31,7 @@ "validator": "SQL_VALIDATOR", "params": { "query": "SELECT MIN(MaxDate) AS latest_date FROM stats", - "condition": "CAST(SUBSTR(latest_date, 1, 4) AS INT) >= YEAR(CURRENT_DATE) - 2" + "condition": "latest_date IS NOT NULL AND CAST(SUBSTR(latest_date, 1, 4) AS INT) >= YEAR(CURRENT_DATE) - 2" } } ] From 1b74500b14512b19e014e4c73fc2c06dbe50cc2a Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Wed, 7 Oct 2026 07:46:59 +0000 Subject: [PATCH 09/15] fixed api issue --- scripts/us_census/pep/us_pep_sex/manifest.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/us_census/pep/us_pep_sex/manifest.json b/scripts/us_census/pep/us_pep_sex/manifest.json index 57568dcd5a..bd2fa05ca9 100644 --- a/scripts/us_census/pep/us_pep_sex/manifest.json +++ b/scripts/us_census/pep/us_pep_sex/manifest.json @@ -11,7 +11,8 @@ "process.py --config_path=unresolved_mcf/us_census/pep/population_estimate_by_sex/skip_url.json" ], "source_files": [ - "gcs_output/us_pep_sex_source_files/*" + "gcs_output/us_pep_sex_source_files/*", + "golden_data/*.csv" ], "import_inputs": [ { From c3f6a5879957264996ad2a0f8154a37a4cb1c7bb Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Wed, 7 Oct 2026 07:58:46 +0000 Subject: [PATCH 10/15] fixed api issue --- .../pep/annual_population/preprocess_test.py | 161 +++++++++++++- scripts/us_census/pep/us_pep_sex/process.py | 2 +- .../us_census/pep/us_pep_sex/process_test.py | 202 +++++++++++++++++- 3 files changed, 362 insertions(+), 3 deletions(-) diff --git a/scripts/us_census/pep/annual_population/preprocess_test.py b/scripts/us_census/pep/annual_population/preprocess_test.py index 07001af4ba..880319612d 100644 --- a/scripts/us_census/pep/annual_population/preprocess_test.py +++ b/scripts/us_census/pep/annual_population/preprocess_test.py @@ -26,9 +26,21 @@ if path not in sys.path: sys.path.append(path) +import json +import re +from unittest.mock import MagicMock, patch +import pandas as pd + # pylint: disable=wrong-import-position # pylint: disable=import-error -from us_census.pep.annual_population.preprocess import process +from us_census.pep.annual_population.preprocess import ( + process, + _process_nationals_2029, + download_with_retry, + _create_retry_session, + add_future_year_urls, + USA, +) from us_census.pep.annual_population.constants import TEST_DATA_DIR # pylint: enable=import-error # pylint: enable=wrong-import-position @@ -109,5 +121,152 @@ def test_create_csv(self): self._actual_csv_data.strip()) +class TestProcessExcel2029(unittest.TestCase): + """Tests for dynamic future year processing in Excel files and POPCHG regex.""" + + def test_process_nationals_2029_float_and_int_headers(self): + """Tests that _process_nationals_2029 correctly parses float and int headers for future years.""" + mock_df = pd.DataFrame({ + 'Geographic Area': [USA, 'Northeast'], + 'April 1, 2020': [331449281, 57000000], + '2020': [331511512, 57100000], + '2021': [332031554, 57200000], + '2022': [333287557, 57300000], + '2023': [334914895, 57400000], + 2024.0: [336000000, + 57500000], # parsed as float by pandas read_excel + 2025: [337000000, 57600000], # parsed as int + }) + with patch('us_census.pep.annual_population.preprocess._load_data_df', + return_value=mock_df): + result = _process_nationals_2029('dummy_path.xlsx') + + self.assertIn('Year', result.columns) + self.assertIn('Count_Person', result.columns) + self.assertIn('Location', result.columns) + years = result['Year'].tolist() + self.assertIn('2024', years) + self.assertIn('2025', years) + self.assertEqual( + result[result['Year'] == '2024']['Count_Person'].iloc[0], + 336000000) + self.assertEqual( + result[result['Year'] == '2025']['Count_Person'].iloc[0], + 337000000) + + def test_popchg_regex_matches_various_years(self): + """Tests that regex pattern matches NST-EST-POPCHG2020_.csv for various years.""" + pattern = r"NST-EST\d{4}-POPCHG2020_\d{4}\.csv" + for year in [2023, 2024, 2025, 2026, 2029]: + filename = f"NST-EST{year}-POPCHG2020_{year}.csv" + self.assertIsNotNone(re.search(pattern, filename), + f"Failed to match {filename}") + + self.assertIsNone(re.search(pattern, "NST-EST2024-POP.csv")) + self.assertIsNone(re.search(pattern, "NST-EST-POPCHG2020.csv")) + + +class TestDownloadAndUrlProbing(unittest.TestCase): + """Tests for download session, retries, HTML error detection, and URL probing.""" + + def test_create_retry_session(self): + """Tests that retry session is properly configured with status codes and backoff.""" + session = _create_retry_session() + self.assertIn('User-Agent', session.headers) + adapter = session.adapters.get('https://') + self.assertIsNotNone(adapter) + self.assertIsNotNone(adapter.max_retries) + self.assertIn(429, adapter.max_retries.status_forcelist) + self.assertIn(500, adapter.max_retries.status_forcelist) + self.assertIn(503, adapter.max_retries.status_forcelist) + + def test_download_with_retry_raises_on_html_error(self): + """Tests that download_with_retry raises ValueError on HTML error page (soft-404).""" + session = MagicMock() + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.headers = {'Content-Type': 'text/html; charset=UTF-8'} + mock_resp.__enter__.return_value = mock_resp + session.get.return_value = mock_resp + + with tempfile.TemporaryDirectory() as tmp_dir: + dest_file = os.path.join(tmp_dir, 'test.xlsx') + with self.assertRaises(ValueError) as ctx: + download_with_retry.__wrapped__( + session, 'https://census.gov/test.xlsx', dest_file) + self.assertIn('HTML error page', str(ctx.exception)) + + def test_download_with_retry_writes_binary_file(self): + """Tests that download_with_retry writes binary data chunks successfully.""" + session = MagicMock() + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.headers = { + 'Content-Type': + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' + } + mock_resp.iter_content.return_value = [b'chunk1', b'chunk2'] + mock_resp.__enter__.return_value = mock_resp + session.get.return_value = mock_resp + + with tempfile.TemporaryDirectory() as tmp_dir: + dest_file = os.path.join(tmp_dir, 'test.xlsx') + download_with_retry.__wrapped__(session, + 'https://census.gov/test.xlsx', + dest_file) + self.assertTrue(os.path.exists(dest_file)) + with open(dest_file, 'rb') as f: + self.assertEqual(f.read(), b'chunk1chunk2') + + def test_add_future_year_urls_gatekeeper_break(self): + """Tests that add_future_year_urls stops scanning when gatekeeper year is found.""" + session = MagicMock() + head_calls = [] + + def mock_head(url, *args, **kwargs): + head_calls.append(url) + resp = MagicMock() + if '2024' in url: + resp.status_code = 200 + resp.headers = {'Content-Type': 'application/vnd.ms-excel'} + else: + resp.status_code = 404 + resp.headers = {'Content-Type': 'text/html'} + return resp + + session.head = mock_head + + with patch( + 'us_census.pep.annual_population.preprocess._create_retry_session', + return_value=session), patch('time.sleep'): + add_future_year_urls() + + years_probed = [url for url in head_calls if '2024' in url] + self.assertTrue(len(years_probed) > 0) + years_2023 = [url for url in head_calls if '2023' in url] + self.assertEqual(len(years_2023), 0) + + def test_add_future_year_urls_fallback_to_baseline(self): + """Tests that add_future_year_urls falls back to baseline year when all future years 404.""" + session = MagicMock() + mock_resp = MagicMock() + mock_resp.status_code = 404 + mock_resp.headers = {'Content-Type': 'text/html'} + session.head.return_value = mock_resp + + import us_census.pep.annual_population.preprocess as prep + with patch( + 'us_census.pep.annual_population.preprocess._create_retry_session', + return_value=session), patch('time.sleep'): + prep.add_future_year_urls() + + download_urls = [ + entry['download_path'] for entry in prep._FILES_TO_DOWNLOAD + ] + self.assertTrue( + any('2020-2025' in url or '2020-2023' in url + for url in download_urls)) + + if __name__ == '__main__': unittest.main() diff --git a/scripts/us_census/pep/us_pep_sex/process.py b/scripts/us_census/pep/us_pep_sex/process.py index 27b7a0e418..26b9d4ce78 100644 --- a/scripts/us_census/pep/us_pep_sex/process.py +++ b/scripts/us_census/pep/us_pep_sex/process.py @@ -600,7 +600,7 @@ def _state_latest(file_path: str) -> pd.DataFrame: df = pd.read_excel(file_path, skiprows=5, skipfooter=7, header=None) current_year = int(max_year) + 1 for year in range(2021, current_year): - if current_year < 2030: + if current_year <= 2030: base_columns.append(f'July{year}Total') base_columns.append(f'July{year}Male') base_columns.append(f'July{year}Female') diff --git a/scripts/us_census/pep/us_pep_sex/process_test.py b/scripts/us_census/pep/us_pep_sex/process_test.py index 26013e7fb0..c81b3a4be8 100644 --- a/scripts/us_census/pep/us_pep_sex/process_test.py +++ b/scripts/us_census/pep/us_pep_sex/process_test.py @@ -19,6 +19,11 @@ import unittest import sys import tempfile +import json +import shutil +from unittest.mock import MagicMock, patch +import numpy as np +import pandas as pd from absl import flags # module_dir is the path to where this test is running from. @@ -29,7 +34,15 @@ sys.path.append(path) # pylint: disable=wrong-import-position -from us_census.pep.us_pep_sex.process import PopulationEstimateBySex +from us_census.pep.us_pep_sex.process import ( + PopulationEstimateBySex, + _state_latest, + _download_single_file, + _create_retry_session, + add_future_year_urls, + download_files, +) +import us_census.pep.us_pep_sex.process as sex_proc # pylint: enable=wrong-import-position TEST_DATASET_DIR = os.path.join(MODULE_DIR, "test_data", "datasets") @@ -111,5 +124,192 @@ def test_create_csv(self): self.actual_csv_data.strip()) +class TestStateLatestVintageBoundary(unittest.TestCase): + """Tests vintage boundary logic in _state_latest.""" + + def test_state_latest_boundary_2029(self): + """Verifies that max_year 2029 (current_year=2030) appends columns up to 2029 without being dropped.""" + header_df = pd.DataFrame( + [['dummy'] * 34, ['dummy'] * 34, + [np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan] + + [2021, 2022, 2023, 2024, 2025, 2026, 2027, 2028, 2029] + + [np.nan] * 18]) + data_row = ['Total'] + [1000] * 33 + data_df = pd.DataFrame([data_row]) + + with patch('pandas.read_excel', side_effect=[header_df, data_df]): + res = _state_latest('dummy_sc-est2029-syasex-01.xlsx') + + self.assertIn('Year', res.columns) + years = res['Year'].unique().tolist() + self.assertIn('2029', years) + self.assertIn('geo_ID', res.columns) + self.assertEqual(res['geo_ID'].iloc[0], 'geoId/01') + + +class TestDynamicFunctionMapping(unittest.TestCase): + """Tests dynamic registration of file-to-function mappings up to year 2029.""" + + def test_file_to_function_mapping_years_2023_to_2029(self): + """Tests that national, state, and county future year files (e.g. 2024, 2029) are dispatched correctly.""" + with tempfile.TemporaryDirectory() as tmp_dir: + for yr in [2024, 2029]: + open(os.path.join(tmp_dir, f"nc-est{yr}-agesex-res.csv"), + 'w').close() + open(os.path.join(tmp_dir, f"sc-est{yr}-syasex-01.xlsx"), + 'w').close() + open(os.path.join(tmp_dir, f"cc-est{yr}-agesex-all.csv"), + 'w').close() + + mock_df = pd.DataFrame({ + 'Year': ['2024'], + 'geo_ID': ['geoId/01'], + 'Count_Person_Male': [10], + 'Count_Person_Female': [10], + 'Measurement_Method': ['CensusPEPSurvey'] + }) + + with patch('us_census.pep.us_pep_sex.process._national_latest', + return_value=mock_df) as mock_nat, \ + patch('us_census.pep.us_pep_sex.process._state_latest', + return_value=mock_df) as mock_st, \ + patch('us_census.pep.us_pep_sex.process._county_latest', + return_value=mock_df) as mock_co: + loader = PopulationEstimateBySex( + tmp_dir, os.path.join(tmp_dir, 'out.csv'), + os.path.join(tmp_dir, 'out.mcf'), + os.path.join(tmp_dir, 'out.tmcf')) + loader.process() + self.assertEqual(mock_nat.call_count, 2) + self.assertEqual(mock_st.call_count, 2) + self.assertEqual(mock_co.call_count, 2) + + +class TestDownloadAndCacheLifecycle(unittest.TestCase): + """Tests for streaming downloads, temp file cleanup, and cache fallback.""" + + def test_download_single_file_success(self): + """Tests that _download_single_file streams chunks to temp file.""" + session = MagicMock() + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.headers = {'Content-Type': 'application/vnd.ms-excel'} + mock_resp.iter_content.return_value = [b'data1', b'data2'] + mock_resp.__enter__.return_value = mock_resp + session.get.return_value = mock_resp + + tmp_path = _download_single_file.__wrapped__( + session, 'https://census.gov/test.xlsx') + self.assertTrue(os.path.exists(tmp_path)) + with open(tmp_path, 'rb') as f: + self.assertEqual(f.read(), b'data1data2') + os.remove(tmp_path) + + def test_download_single_file_html_error(self): + """Tests that _download_single_file raises ValueError on HTML error response.""" + session = MagicMock() + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.headers = {'Content-Type': 'text/html; charset=UTF-8'} + mock_resp.__enter__.return_value = mock_resp + session.get.return_value = mock_resp + + with self.assertRaises(ValueError) as ctx: + _download_single_file.__wrapped__(session, + 'https://census.gov/test.xlsx') + self.assertIn('HTML error page', str(ctx.exception)) + + def test_download_single_file_cleanup_on_exception(self): + """Tests that temporary files are deleted if an error occurs mid-stream.""" + session = MagicMock() + mock_resp = MagicMock() + mock_resp.headers = {'Content-Type': 'application/octet-stream'} + mock_resp.__enter__.return_value = mock_resp + + created_files = [] + original_named_temp = tempfile.NamedTemporaryFile + + def track_temp(*args, **kwargs): + tf = original_named_temp(*args, **kwargs) + created_files.append(tf.name) + return tf + + def failing_iter(*args, **kwargs): + yield b'chunk1' + raise IOError('Connection aborted') + + mock_resp.iter_content = failing_iter + session.get.return_value = mock_resp + + with patch('tempfile.NamedTemporaryFile', side_effect=track_temp): + with self.assertRaises(IOError): + _download_single_file.__wrapped__(session, + 'https://census.gov/broken') + + self.assertEqual(len(created_files), 1) + self.assertFalse(os.path.exists(created_files[0])) + + def test_download_files_cache_fallback(self): + """Tests that download_files falls back to cached persistent file if network download fails.""" + with tempfile.TemporaryDirectory() as tmp_dir: + gcs_persistent = os.path.join(tmp_dir, 'gcs_persistent') + input_dir = os.path.join(tmp_dir, 'input_files') + os.makedirs(gcs_persistent) + os.makedirs(input_dir) + + cached_file = os.path.join(gcs_persistent, + 'nc-est2024-agesex-res.csv') + with open(cached_file, 'w') as f: + f.write('cached_content') + + # Force file to appear expired to trigger download attempt + os.utime(cached_file, (0, 0)) + + files_to_download = [{ + 'download_path': + 'https://census.gov/nc-est2024-agesex-res.csv' + }] + + with patch.object(sex_proc, '_GCS_FOLDER_PERSISTENT_PATH', gcs_persistent), \ + patch.object(sex_proc, '_INPUT_FILE_PATH', input_dir), \ + patch.object(sex_proc, '_FILES_TO_DOWNLOAD', files_to_download), \ + patch.object(sex_proc, '_download_single_file', side_effect=IOError('Network error')), \ + patch('time.sleep'): + sex_proc.download_files.__wrapped__() + + dest_file = os.path.join(input_dir, 'nc-est2024-agesex-res.csv') + self.assertTrue(os.path.exists(dest_file)) + with open(dest_file) as f: + self.assertEqual(f.read(), 'cached_content') + self.assertTrue(files_to_download[0]['is_downloaded']) + + def test_add_future_year_urls_gatekeeper_break(self): + """Tests that add_future_year_urls gatekeeper loop breaks immediately upon finding the first valid year.""" + session = MagicMock() + head_calls = [] + + def mock_head(url, *args, **kwargs): + head_calls.append(url) + resp = MagicMock() + if '2024' in url: + resp.status_code = 200 + resp.headers = {'Content-Type': 'text/csv'} + else: + resp.status_code = 404 + resp.headers = {'Content-Type': 'text/html'} + return resp + + session.head = mock_head + + with patch.object(sex_proc, 'extract_gcs_info', return_value=('bucket', 'path')), \ + patch.object(sex_proc, 'fetch_skip_urls_from_gcs', return_value=set()), \ + patch.object(sex_proc, '_create_retry_session', return_value=session), \ + patch('time.sleep'): + sex_proc.add_future_year_urls() + + self.assertTrue(any('2024' in url for url in head_calls)) + self.assertFalse(any('2023' in url for url in head_calls)) + + if __name__ == "__main__": unittest.main() From e50b90f3008ed106e097318382f1664f351f73b5 Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Wed, 7 Oct 2026 11:54:09 +0000 Subject: [PATCH 11/15] fixed api issue --- .../us_census/pep/annual_population/validation_config.json | 4 ++-- scripts/us_census/pep/us_pep_sex/validation_config.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/us_census/pep/annual_population/validation_config.json b/scripts/us_census/pep/annual_population/validation_config.json index 4f448c54d7..96ab51bf96 100644 --- a/scripts/us_census/pep/annual_population/validation_config.json +++ b/scripts/us_census/pep/annual_population/validation_config.json @@ -30,8 +30,8 @@ "description": "Verifies that all StatVars meet freshness requirements (within 2 years of current year).", "validator": "SQL_VALIDATOR", "params": { - "query": "SELECT MIN(MaxDate) AS latest_date FROM stats", - "condition": "latest_date IS NOT NULL AND CAST(SUBSTR(latest_date, 1, 4) AS INT) >= YEAR(CURRENT_DATE) - 2" + "query": "SELECT CAST(MIN(MaxDate) AS VARCHAR) AS latest_date FROM stats", + "condition": "latest_date IS NOT NULL AND CAST(SUBSTR(CAST(latest_date AS VARCHAR), 1, 4) AS INT) >= YEAR(CURRENT_DATE) - 2" } } ] diff --git a/scripts/us_census/pep/us_pep_sex/validation_config.json b/scripts/us_census/pep/us_pep_sex/validation_config.json index 586e23600f..daaaea080f 100644 --- a/scripts/us_census/pep/us_pep_sex/validation_config.json +++ b/scripts/us_census/pep/us_pep_sex/validation_config.json @@ -30,8 +30,8 @@ "description": "Verifies that all StatVars meet freshness requirements (within 2 years of current year).", "validator": "SQL_VALIDATOR", "params": { - "query": "SELECT MIN(MaxDate) AS latest_date FROM stats", - "condition": "latest_date IS NOT NULL AND CAST(SUBSTR(latest_date, 1, 4) AS INT) >= YEAR(CURRENT_DATE) - 2" + "query": "SELECT CAST(MIN(MaxDate) AS VARCHAR) AS latest_date FROM stats", + "condition": "latest_date IS NOT NULL AND CAST(SUBSTR(CAST(latest_date AS VARCHAR), 1, 4) AS INT) >= YEAR(CURRENT_DATE) - 2" } } ] From d92c79d9837c7d76ee888378a2b7755a81c6ca4b Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Mon, 12 Oct 2026 06:14:43 +0000 Subject: [PATCH 12/15] fixed lint error --- .../pep/annual_population/preprocess.py | 21 ++++++------ .../pep/annual_population/preprocess_test.py | 13 ++++---- scripts/us_census/pep/us_pep_sex/process.py | 32 ++++++++----------- .../us_census/pep/us_pep_sex/process_test.py | 3 +- 4 files changed, 31 insertions(+), 38 deletions(-) diff --git a/scripts/us_census/pep/annual_population/preprocess.py b/scripts/us_census/pep/annual_population/preprocess.py index 641ab8c9ad..8dfe1288e6 100644 --- a/scripts/us_census/pep/annual_population/preprocess.py +++ b/scripts/us_census/pep/annual_population/preprocess.py @@ -366,8 +366,8 @@ def _process_nationals_1990_1999(ip_file: str) -> pd.DataFrame: "Count_Person_Female" ] data_df.columns = df_cols - data_df = data_df[(data_df["Age"] == "All Age") - & (data_df["Year"].str.startswith("July"))].reset_index( + data_df = data_df[(data_df["Age"] == "All Age") & + (data_df["Year"].str.startswith("July"))].reset_index( drop=True) data_df["Year"] = data_df["Year"].str.replace("July 1, ", "") data_df = data_df.drop( @@ -495,7 +495,8 @@ def _process_nationals_2029(file_path: str) -> pd.DataFrame: unpivot_cols = ["2022", "2023"] # extend df_cols & unpivot_cols with all years > 2023 newly_added_years = [ - str(int(x)) for x in data_df.columns + str(int(x)) + for x in data_df.columns if isinstance(x, (int, float)) and not pd.isna(x) and int(x) > 2023 ] df_cols.extend(newly_added_years) @@ -549,7 +550,8 @@ def _process_states_2029(file_path: str) -> pd.DataFrame: header=0).iloc[2:3].values.flatten().tolist() if year_column_row: newly_added_years = [ - str(int(x)) for x in year_column_row + str(int(x)) + for x in year_column_row if isinstance(x, (int, float)) and not pd.isna(x) and int(x) > 2023 ] @@ -851,8 +853,7 @@ def _process_city_1990_1999(file_path: str) -> pd.DataFrame: data[1] = "06000" loc = "geoId/" + f"{int(data[0]):02d}" \ + f"{int(data[1]):05d}" - for year, val in dict(zip(cols, - data[-len(cols):])).items(): + for year, val in dict(zip(cols, data[-len(cols):])).items(): outfile.write(f"{year},{loc},{val}\n") data_df = pd.read_csv("out.csv", header=0) os.remove("out.csv") @@ -1038,7 +1039,7 @@ def process(input_path, cleaned_csv_file_path: str, mcf_file_path: str, county_df = _process_counties(file) data_df = pd.concat([nat_df, county_df]) elif file_name in ["e7079co.txt", "99c8_00.txt" - ] or "co-est" in file_name: + ] or "co-est" in file_name: data_df = _process_counties(file) elif file_name in [ 'su-99-7_us.txt', "sub-est2010-alt.csv", @@ -1121,8 +1122,7 @@ def add_future_year_urls(): for future_year in range(2030, 2022, -1): url_to_check = url.format(YEAR=future_year) try: - logging.info( - f"checking future url if available {url_to_check}") + logging.info(f"checking future url if available {url_to_check}") check_url = session.head(url_to_check, allow_redirects=True, timeout=30) @@ -1224,8 +1224,7 @@ def main(_): # Defining Output file names data_file_path = os.path.join(_MODULE_DIR, OUTPUT_DIR) os.makedirs(data_file_path, exist_ok=True) - cleaned_csv_path = os.path.join(data_file_path, - "usa_annual_population.csv") + cleaned_csv_path = os.path.join(data_file_path, "usa_annual_population.csv") mcf_path = os.path.join(data_file_path, "usa_annual_population.mcf") tmcf_path = os.path.join(data_file_path, "usa_annual_population.tmcf") diff --git a/scripts/us_census/pep/annual_population/preprocess_test.py b/scripts/us_census/pep/annual_population/preprocess_test.py index 880319612d..cec87a0816 100644 --- a/scripts/us_census/pep/annual_population/preprocess_test.py +++ b/scripts/us_census/pep/annual_population/preprocess_test.py @@ -148,11 +148,9 @@ def test_process_nationals_2029_float_and_int_headers(self): self.assertIn('2024', years) self.assertIn('2025', years) self.assertEqual( - result[result['Year'] == '2024']['Count_Person'].iloc[0], - 336000000) + result[result['Year'] == '2024']['Count_Person'].iloc[0], 336000000) self.assertEqual( - result[result['Year'] == '2025']['Count_Person'].iloc[0], - 337000000) + result[result['Year'] == '2025']['Count_Person'].iloc[0], 337000000) def test_popchg_regex_matches_various_years(self): """Tests that regex pattern matches NST-EST-POPCHG2020_.csv for various years.""" @@ -192,8 +190,9 @@ def test_download_with_retry_raises_on_html_error(self): with tempfile.TemporaryDirectory() as tmp_dir: dest_file = os.path.join(tmp_dir, 'test.xlsx') with self.assertRaises(ValueError) as ctx: - download_with_retry.__wrapped__( - session, 'https://census.gov/test.xlsx', dest_file) + download_with_retry.__wrapped__(session, + 'https://census.gov/test.xlsx', + dest_file) self.assertIn('HTML error page', str(ctx.exception)) def test_download_with_retry_writes_binary_file(self): @@ -203,7 +202,7 @@ def test_download_with_retry_writes_binary_file(self): mock_resp.status_code = 200 mock_resp.headers = { 'Content-Type': - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' } mock_resp.iter_content.return_value = [b'chunk1', b'chunk2'] mock_resp.__enter__.return_value = mock_resp diff --git a/scripts/us_census/pep/us_pep_sex/process.py b/scripts/us_census/pep/us_pep_sex/process.py index 26b9d4ce78..f5d31a8882 100644 --- a/scripts/us_census/pep/us_pep_sex/process.py +++ b/scripts/us_census/pep/us_pep_sex/process.py @@ -209,8 +209,8 @@ def _national_1990_2000(file_path: str) -> pd.DataFrame: 'Year', 'Age', 'Total', 'Count_Person_Male', 'Count_Person_Female' ] # total age is required as we are bring age in a seperate import - df = df[(df["Age"] == "All Age") - & (df["Year"].str.startswith("July"))].reset_index(drop=True) + df = df[(df["Age"] == "All Age") & + (df["Year"].str.startswith("July"))].reset_index(drop=True) df["Year"] = df["Year"].str.replace("July 1, ", "") # dropping unwanted columns df = df.drop(columns=df.columns.difference( @@ -560,8 +560,8 @@ def _state_2010_2020(file_path: str) -> pd.DataFrame: df = df.drop(columns=df.columns.difference( ['Year', 'Count_Person_Male', 'Count_Person_Female', 'geo_ID'])) - df = df[(df['Year'] != 'April2010Census') - & (df['Year'] != 'April2010Estimate') & + df = df[(df['Year'] != 'April2010Census') & + (df['Year'] != 'April2010Estimate') & (df['Year'] != 'April2020') & (df['Year'] != '2020')] df['Measurement_Method'] = 'dcAggregate/CensusPEPSurvey_PartialAggregate' return df @@ -882,8 +882,8 @@ def _county_2010_2020(file_path: str) -> pd.DataFrame: df = df.drop(columns=df.columns.difference( ['Year', 'Count_Person_Male', 'Count_Person_Female', 'geo_ID'])) - df = df[(df['Year'] != 'April2010Census') - & (df['Year'] != 'April2010Estimate') & + df = df[(df['Year'] != 'April2010Census') & + (df['Year'] != 'April2010Estimate') & (df['Year'] != 'April2020')] df['Measurement_Method'] = 'CensusPEPSurvey' return df @@ -930,8 +930,8 @@ def _county_latest(file_path: str) -> pd.DataFrame: df = df.drop(columns=df.columns.difference( ['Year', 'Count_Person_Male', 'Count_Person_Female', 'geo_ID'])) - df = df[(df['Year'] != 'April2020Estimate') - & (df['Year'] != 'July2020')] + df = df[(df['Year'] != 'April2020Estimate') & + (df['Year'] != 'July2020')] df['Measurement_Method'] = 'CensusPEPSurvey' return df @@ -1196,7 +1196,7 @@ def add_future_year_urls(): if (url["download_path"] not in skip_urls # not suspicious, keep it or (url["download_path"] in skip_urls and is_valid_url( session, url["download_path"])) # suspicious, but passes check - ) + ) ] logging.info( @@ -1250,8 +1250,7 @@ def add_future_year_urls(): allow_redirects=True, timeout=15) time.sleep(0.1) - content_type = check_url.headers.get( - 'Content-Type', '') + content_type = check_url.headers.get('Content-Type', '') if check_url.status_code == 200 and 'html' not in content_type.lower( ): _FILES_TO_DOWNLOAD.append( @@ -1310,14 +1309,12 @@ def cleanup(): active_files = set() if _FILES_TO_DOWNLOAD: active_files = { - _get_target_file_name(item) - for item in _FILES_TO_DOWNLOAD + _get_target_file_name(item) for item in _FILES_TO_DOWNLOAD } for file_name in os.listdir(_GCS_FOLDER_PERSISTENT_PATH): file_path = os.path.join(_GCS_FOLDER_PERSISTENT_PATH, file_name) if os.path.isfile(file_path): - file_age = (time.time() - os.path.getmtime(file_path)) / (24 * - 3600) + file_age = (time.time() - os.path.getmtime(file_path)) / (24 * 3600) if file_age > _TTL_DAYS and file_name not in active_files: logging.info(f"Cleaning up old file: {file_name}") os.remove(file_path) @@ -1472,9 +1469,8 @@ def main(_): cleanup() if download_status and (mode == "" or mode == "process"): try: - loader = PopulationEstimateBySex(_INPUT_FILE_PATH, - cleaned_csv_path, mcf_path, - tmcf_path) + loader = PopulationEstimateBySex(_INPUT_FILE_PATH, cleaned_csv_path, + mcf_path, tmcf_path) loader.process() # The persistent folder is intentionally kept to allow for TTL caching. diff --git a/scripts/us_census/pep/us_pep_sex/process_test.py b/scripts/us_census/pep/us_pep_sex/process_test.py index c81b3a4be8..2dcd71f0b3 100644 --- a/scripts/us_census/pep/us_pep_sex/process_test.py +++ b/scripts/us_census/pep/us_pep_sex/process_test.py @@ -266,8 +266,7 @@ def test_download_files_cache_fallback(self): os.utime(cached_file, (0, 0)) files_to_download = [{ - 'download_path': - 'https://census.gov/nc-est2024-agesex-res.csv' + 'download_path': 'https://census.gov/nc-est2024-agesex-res.csv' }] with patch.object(sex_proc, '_GCS_FOLDER_PERSISTENT_PATH', gcs_persistent), \ From 603d955b818fa2a6b39fbaa2e62fb5a6bb717035 Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Wed, 14 Oct 2026 05:21:46 +0000 Subject: [PATCH 13/15] resolved comments --- .../pep/annual_population/preprocess.py | 131 +++++--- .../pep/annual_population/preprocess_test.py | 141 ++++++--- .../annual_population/validation_config.json | 16 +- .../us_census/pep/us_pep_sex/manifest.json | 2 +- scripts/us_census/pep/us_pep_sex/process.py | 296 ++++++++++-------- .../us_census/pep/us_pep_sex/process_test.py | 116 ++++--- .../pep/us_pep_sex/validation_config.json | 16 +- 7 files changed, 471 insertions(+), 247 deletions(-) diff --git a/scripts/us_census/pep/annual_population/preprocess.py b/scripts/us_census/pep/annual_population/preprocess.py index 8dfe1288e6..f711b9c911 100644 --- a/scripts/us_census/pep/annual_population/preprocess.py +++ b/scripts/us_census/pep/annual_population/preprocess.py @@ -43,6 +43,7 @@ import requests import time import re +import tempfile from datetime import datetime as dt warnings.filterwarnings('ignore') @@ -68,6 +69,7 @@ def _define_flags(): _INPUT_FILE_PATH = os.path.join(_MODULE_DIR, 'input_files') os.makedirs(_INPUT_FILE_PATH, exist_ok=True) excel_file_name_pattern = r"NST-EST(\d{4})-POP\.xlsx" +popchg_file_name_pattern = r"NST-EST\d{4}-POPCHG2020_\d{4}\.csv" sys.path.insert(1, _MODULE_DIR) # pylint: disable=wrong-import-position # pylint: disable=import-error @@ -488,19 +490,40 @@ def _process_nationals_2021(file_path: str) -> pd.DataFrame: return data_df +def _extract_future_years(values, min_year: int = 2023) -> list: + """Extracts year strings > min_year from header values (handling int, float, and str).""" + years = [] + for x in values: + if pd.isna(x): + continue + if isinstance(x, (int, float)): + yr = int(x) + elif isinstance(x, str): + s = x.strip() + if re.fullmatch(r"\d{4}(?:\.0)?", s): + yr = int(float(s)) + else: + continue + else: + continue + if yr > min_year: + years.append(str(yr)) + return years + + def _process_nationals_2029(file_path: str) -> pd.DataFrame: data_df = pd.DataFrame() data_df = _load_data_df(path=file_path, file_format="xlsx", header=3) df_cols = ["Location", "042020", "072020", "072021", "2022", "2023"] unpivot_cols = ["2022", "2023"] # extend df_cols & unpivot_cols with all years > 2023 - newly_added_years = [ - str(int(x)) - for x in data_df.columns - if isinstance(x, (int, float)) and not pd.isna(x) and int(x) > 2023 - ] + newly_added_years = _extract_future_years(data_df.columns, min_year=2023) df_cols.extend(newly_added_years) unpivot_cols.extend(newly_added_years) + if len(df_cols) != len(data_df.columns): + raise ValueError( + f"Column count mismatch in {file_path}: expected {len(df_cols)} columns ({df_cols}), " + f"found {len(data_df.columns)} columns ({list(data_df.columns)})") data_df.columns = df_cols data_df = data_df[data_df["Location"] == USA] data_df = _unpivot_data_df(data_df, ["Location"], unpivot_cols) @@ -543,24 +566,21 @@ def _process_states_2029(file_path: str) -> pd.DataFrame: "Region", "ignore_col1", "ignore_col2", "ignore_col3", "2022", "2023" ] unpivot_cols = ["2022", "2023"] - try: - # extend df_cols & unpivot_cols with all years > 2023 - year_column_row = _load_data_df( - path=file_path, file_format="xlsx", - header=0).iloc[2:3].values.flatten().tolist() - if year_column_row: - newly_added_years = [ - str(int(x)) - for x in year_column_row - if isinstance(x, (int, - float)) and not pd.isna(x) and int(x) > 2023 - ] - df_cols.extend(newly_added_years) - unpivot_cols.extend(newly_added_years) - except (ValueError, IndexError, KeyError, TypeError) as e: - logging.warning( - f"Could not extract dynamic year columns from {file_path}: {e}") + # extend df_cols & unpivot_cols with all years > 2023 + header_df = _load_data_df(path=file_path, file_format="xlsx", header=0) + if len(header_df) <= 2: + raise ValueError( + f"Unable to extract dynamic year columns from {file_path}: header row 2 missing" + ) + year_column_row = header_df.iloc[2:3].values.flatten().tolist() + newly_added_years = _extract_future_years(year_column_row, min_year=2023) + df_cols.extend(newly_added_years) + unpivot_cols.extend(newly_added_years) data_df = _load_data_df(path=file_path, file_format="xlsx", header=8) + if len(df_cols) != len(data_df.columns): + raise ValueError( + f"Column count mismatch in {file_path}: expected {len(df_cols)} columns ({df_cols}), " + f"found {len(data_df.columns)} columns") data_df.columns = df_cols data_df["Region"] = data_df["Region"].str.replace(".", "", regex=False) @@ -1002,7 +1022,7 @@ def process(input_path, cleaned_csv_file_path: str, mcf_file_path: str, _process_nationals_2010_2020(file, op_file) data_df = _load_data_df(op_file, "csv", 0) os.remove(op_file) - elif re.search(r"NST-EST\d{4}-POPCHG2020_\d{4}\.csv", file_name): + elif re.search(popchg_file_name_pattern, file_name): _process_nationals_2020_2029(file, op_file) data_df = _load_data_df(op_file, "csv", 0) os.remove(op_file) @@ -1049,10 +1069,9 @@ def process(input_path, cleaned_csv_file_path: str, mcf_file_path: str, if not data_df.empty: processed_count += 1 - # data_df['len'] = data_df['Location'].astype(str).map(len) - # geo_code_len = data_df['len'].unique() - # if 13 in geo_code_len: - # pass + vintage_matches = re.findall(r"(?:19|20)\d{2}", file_name) + data_df["_vintage"] = max([int(y) for y in vintage_matches], + default=0) final_df = pd.concat([final_df, data_df], axis=0) else: logging.error(f"Failed to process {file}") @@ -1067,8 +1086,9 @@ def process(input_path, cleaned_csv_file_path: str, mcf_file_path: str, f"Mismatch between files expected ({total_files_to_process}) and processed ({processed_count})" ) final_df["Year"] = final_df["Year"].astype("int") - final_df = final_df.sort_values(by=["Location", "Year"]) - final_df = final_df.drop_duplicates(["Year", "Location"]) + final_df = final_df.sort_values(by=["Location", "Year", "_vintage"], + kind="mergesort") + final_df = final_df.drop_duplicates(["Year", "Location"], keep="last") final_df["Count_Person"] = final_df["Count_Person"].astype("int") final_df[["Year", "Location", "Count_Person"]].to_csv(cleaned_csv_file_path, index=False) @@ -1085,7 +1105,7 @@ def _create_retry_session() -> requests.Session: retries = Retry( total=5, backoff_factor=2, - status_forcelist=[429, 500, 502, 503, 504, 520], + status_forcelist=[408, 429, 500, 502, 503, 504, 520], allowed_methods=["HEAD", "GET"], respect_retry_after_header=True, ) @@ -1095,6 +1115,18 @@ def _create_retry_session() -> requests.Session: return session +@retry(tries=3, delay=2, backoff=2, exceptions=(ValueError,)) +def _head_with_html_retry(session: requests.Session, + url: str, + timeout: int = 30) -> requests.Response: + """Performs a HEAD request and retries if HTTP 200 returns an HTML error page.""" + response = session.head(url, allow_redirects=True, timeout=timeout) + content_type = response.headers.get('Content-Type', '') + if response.status_code == 200 and 'html' in content_type.lower(): + raise ValueError(f"Server returned HTTP 200 HTML response for {url}") + return response + + def add_future_year_urls(): """ This method will generate future URLs specified in the urls_to_scan object. If valid URL, it will append to the _FILES_TO_DOWNLOAD global variable. @@ -1116,16 +1148,16 @@ def add_future_year_urls(): 2023), ] session = _create_retry_session() - # This method will get the latest year URLs available for the years 2023 to 2030 + # This method will get the latest year URLs available for the years baseline_year to 2030 for url, baseline_year in urls_to_scan: found = False - for future_year in range(2030, 2022, -1): + for future_year in range(2030, baseline_year - 1, -1): url_to_check = url.format(YEAR=future_year) try: logging.info(f"checking future url if available {url_to_check}") - check_url = session.head(url_to_check, - allow_redirects=True, - timeout=30) + check_url = _head_with_html_retry(session, + url_to_check, + timeout=30) time.sleep(0.2) content_type = check_url.headers.get('Content-Type', '') if check_url.status_code == 200 and 'html' not in content_type.lower( @@ -1134,7 +1166,7 @@ def add_future_year_urls(): _FILES_TO_DOWNLOAD.append({"download_path": url_to_check}) found = True break - except requests.RequestException as e: + except (requests.RequestException, ValueError) as e: logging.error(f"URL is not accessible {url_to_check}: {e}") if not found: fallback_url = url.format(YEAR=baseline_year) @@ -1144,10 +1176,7 @@ def add_future_year_urls(): _FILES_TO_DOWNLOAD.append({"download_path": fallback_url}) -@retry(tries=5, - delay=5, - backoff=2, - exceptions=(requests.RequestException, ValueError)) +@retry(tries=3, delay=5, backoff=2, exceptions=(ValueError,)) def download_with_retry(session: requests.Session, url: str, dest_path: str): logging.info(f"Downloading file from URL: {url} -> {dest_path}") with session.get(url, stream=True, timeout=120) as response: @@ -1155,10 +1184,24 @@ def download_with_retry(session: requests.Session, url: str, dest_path: str): content_type = response.headers.get('Content-Type', '') if 'html' in content_type.lower(): raise ValueError(f"Server returned HTML error page for URL: {url}") - with open(dest_path, 'wb') as f: - for chunk in response.iter_content(chunk_size=65536): - if chunk: - f.write(chunk) + tmp_file_path = None + try: + dest_dir = os.path.dirname(dest_path) or '.' + os.makedirs(dest_dir, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=dest_dir, + delete=False) as tmp_file: + tmp_file_path = tmp_file.name + for chunk in response.iter_content(chunk_size=65536): + if chunk: + tmp_file.write(chunk) + os.replace(tmp_file_path, dest_path) + tmp_file_path = None + finally: + if tmp_file_path and os.path.exists(tmp_file_path): + try: + os.remove(tmp_file_path) + except OSError: + pass logging.info(f"Downloaded file: {dest_path}") diff --git a/scripts/us_census/pep/annual_population/preprocess_test.py b/scripts/us_census/pep/annual_population/preprocess_test.py index cec87a0816..04eb0d7d86 100644 --- a/scripts/us_census/pep/annual_population/preprocess_test.py +++ b/scripts/us_census/pep/annual_population/preprocess_test.py @@ -26,7 +26,6 @@ if path not in sys.path: sys.path.append(path) -import json import re from unittest.mock import MagicMock, patch import pandas as pd @@ -36,11 +35,14 @@ from us_census.pep.annual_population.preprocess import ( process, _process_nationals_2029, + _process_states_2029, download_with_retry, _create_retry_session, add_future_year_urls, + popchg_file_name_pattern, USA, ) +import us_census.pep.annual_population.preprocess as prep from us_census.pep.annual_population.constants import TEST_DATA_DIR # pylint: enable=import-error # pylint: enable=wrong-import-position @@ -122,10 +124,10 @@ def test_create_csv(self): class TestProcessExcel2029(unittest.TestCase): - """Tests for dynamic future year processing in Excel files and POPCHG regex.""" + """Tests for dynamic future year processing in Excel files, POPCHG regex, and vintage precedence.""" - def test_process_nationals_2029_float_and_int_headers(self): - """Tests that _process_nationals_2029 correctly parses float and int headers for future years.""" + def test_process_nationals_2029_float_int_and_str_headers(self): + """Tests that _process_nationals_2029 correctly parses float, int, and str headers for future years.""" mock_df = pd.DataFrame({ 'Geographic Area': [USA, 'Northeast'], 'April 1, 2020': [331449281, 57000000], @@ -135,7 +137,7 @@ def test_process_nationals_2029_float_and_int_headers(self): '2023': [334914895, 57400000], 2024.0: [336000000, 57500000], # parsed as float by pandas read_excel - 2025: [337000000, 57600000], # parsed as int + '2025': [337000000, 57600000], # parsed as str }) with patch('us_census.pep.annual_population.preprocess._load_data_df', return_value=mock_df): @@ -152,16 +154,68 @@ def test_process_nationals_2029_float_and_int_headers(self): self.assertEqual( result[result['Year'] == '2025']['Count_Person'].iloc[0], 337000000) - def test_popchg_regex_matches_various_years(self): - """Tests that regex pattern matches NST-EST-POPCHG2020_.csv for various years.""" - pattern = r"NST-EST\d{4}-POPCHG2020_\d{4}\.csv" + def test_process_states_2029_column_length_mismatch_raises(self): + """Tests that _process_states_2029 fails fast with ValueError when extracted headers do not match data columns.""" + header_df = pd.DataFrame([['a'] * 8, ['b'] * 8, [None] * 8]) + data_df = pd.DataFrame([['.Alabama', 1, 2, 3, 4, 5, 6, 7]]) + with patch('us_census.pep.annual_population.preprocess._load_data_df', + side_effect=[header_df, data_df]): + with self.assertRaises(ValueError): + _process_states_2029('dummy_path.xlsx') + + def test_popchg_regex_and_vintage_precedence_in_process(self): + """Tests popchg_file_name_pattern and verifies newer vintage NST-EST2025-POP.xlsx takes precedence over NST-EST2023-POPCHG2020_2023.csv.""" for year in [2023, 2024, 2025, 2026, 2029]: filename = f"NST-EST{year}-POPCHG2020_{year}.csv" - self.assertIsNotNone(re.search(pattern, filename), + self.assertIsNotNone(re.search(popchg_file_name_pattern, filename), f"Failed to match {filename}") - self.assertIsNone(re.search(pattern, "NST-EST2024-POP.csv")) - self.assertIsNone(re.search(pattern, "NST-EST-POPCHG2020.csv")) + self.assertIsNone( + re.search(popchg_file_name_pattern, "NST-EST2024-POP.csv")) + self.assertIsNone( + re.search(popchg_file_name_pattern, "NST-EST-POPCHG2020.csv")) + + with tempfile.TemporaryDirectory() as tmp_dir: + in_dir = os.path.join(tmp_dir, 'input') + out_dir = os.path.join(tmp_dir, 'output') + os.makedirs(in_dir) + os.makedirs(out_dir) + + popchg_path = os.path.join(in_dir, + 'NST-EST2023-POPCHG2020_2023.csv') + with open(popchg_path, 'w') as f: + f.write( + 'NAME,POPESTIMATE2020,POPESTIMATE2021,POPESTIMATE2022,POPESTIMATE2023\n' + 'United States,331500000,332000000,333271411,334914895\n') + + excel_path = os.path.join(in_dir, 'NST-EST2025-POP.xlsx') + open(excel_path, 'w').close() + + nat_2025_df = pd.DataFrame({ + 'Year': ['2022', '2023', '2024', '2025'], + 'Location': ['country/USA'] * 4, + 'Count_Person': [334017321, 336806231, 340003797, 341784857], + }) + state_2025_df = pd.DataFrame({ + 'Year': ['2025'], + 'Location': ['geoId/01'], + 'Count_Person': [5157699], + }) + + csv_out = os.path.join(out_dir, 'usa_annual_population.csv') + mcf_out = os.path.join(out_dir, 'usa_annual_population.mcf') + tmcf_out = os.path.join(out_dir, 'usa_annual_population.tmcf') + + with patch.object(prep, '_process_nationals_2029', return_value=nat_2025_df), \ + patch.object(prep, '_process_states_2029', return_value=state_2025_df): + process(in_dir, csv_out, mcf_out, tmcf_out, False) + + out_df = pd.read_csv(csv_out) + usa_df = out_df[out_df['Location'] == 'country/USA'].set_index( + 'Year') + self.assertEqual(usa_df.loc[2022, 'Count_Person'], 334017321) + self.assertEqual(usa_df.loc[2023, 'Count_Person'], 336806231) + self.assertEqual(usa_df.loc[2025, 'Count_Person'], 341784857) class TestDownloadAndUrlProbing(unittest.TestCase): @@ -194,9 +248,10 @@ def test_download_with_retry_raises_on_html_error(self): 'https://census.gov/test.xlsx', dest_file) self.assertIn('HTML error page', str(ctx.exception)) + self.assertFalse(os.path.exists(dest_file)) def test_download_with_retry_writes_binary_file(self): - """Tests that download_with_retry writes binary data chunks successfully.""" + """Tests that download_with_retry writes binary data chunks atomically.""" session = MagicMock() mock_resp = MagicMock() mock_resp.status_code = 200 @@ -218,14 +273,14 @@ def test_download_with_retry_writes_binary_file(self): self.assertEqual(f.read(), b'chunk1chunk2') def test_add_future_year_urls_gatekeeper_break(self): - """Tests that add_future_year_urls stops scanning when gatekeeper year is found.""" + """Tests that add_future_year_urls stops scanning when a valid future year is found.""" session = MagicMock() head_calls = [] def mock_head(url, *args, **kwargs): head_calls.append(url) resp = MagicMock() - if '2024' in url: + if '2026' in url: resp.status_code = 200 resp.headers = {'Content-Type': 'application/vnd.ms-excel'} else: @@ -235,35 +290,51 @@ def mock_head(url, *args, **kwargs): session.head = mock_head - with patch( - 'us_census.pep.annual_population.preprocess._create_retry_session', - return_value=session), patch('time.sleep'): + with patch.object(prep, '_FILES_TO_DOWNLOAD', []), \ + patch('us_census.pep.annual_population.preprocess._create_retry_session', + return_value=session), \ + patch('time.sleep'): add_future_year_urls() + download_urls = [ + entry['download_path'] for entry in prep._FILES_TO_DOWNLOAD + ] - years_probed = [url for url in head_calls if '2024' in url] + years_probed = [url for url in head_calls if '2026' in url] self.assertTrue(len(years_probed) > 0) - years_2023 = [url for url in head_calls if '2023' in url] - self.assertEqual(len(years_2023), 0) + years_2025 = [url for url in head_calls if '2025' in url] + self.assertEqual(len(years_2025), 0) + self.assertTrue(any('2020-2026' in url for url in download_urls)) def test_add_future_year_urls_fallback_to_baseline(self): - """Tests that add_future_year_urls falls back to baseline year when all future years 404.""" + """Tests that add_future_year_urls stops at baseline_year and falls back without downgrading to older years.""" session = MagicMock() - mock_resp = MagicMock() - mock_resp.status_code = 404 - mock_resp.headers = {'Content-Type': 'text/html'} - session.head.return_value = mock_resp - - import us_census.pep.annual_population.preprocess as prep - with patch( - 'us_census.pep.annual_population.preprocess._create_retry_session', - return_value=session), patch('time.sleep'): - prep.add_future_year_urls() + head_calls = [] - download_urls = [ - entry['download_path'] for entry in prep._FILES_TO_DOWNLOAD - ] + def mock_head(url, *args, **kwargs): + head_calls.append(url) + resp = MagicMock() + resp.status_code = 404 + resp.headers = {'Content-Type': 'text/html'} + return resp + + session.head = mock_head + + with patch.object(prep, '_FILES_TO_DOWNLOAD', []), \ + patch('us_census.pep.annual_population.preprocess._create_retry_session', + return_value=session), \ + patch('time.sleep'): + prep.add_future_year_urls() + download_urls = [ + entry['download_path'] for entry in prep._FILES_TO_DOWNLOAD + ] + + # Ensure 2025-baseline templates never probed 2024 + self.assertFalse( + any('NST-EST2024-POP.xlsx' in url for url in head_calls)) + self.assertTrue(any('NST-EST2025-POP.xlsx' in url + for url in download_urls)) self.assertTrue( - any('2020-2025' in url or '2020-2023' in url + any('NST-EST2023-POPCHG2020_2023.csv' in url for url in download_urls)) diff --git a/scripts/us_census/pep/annual_population/validation_config.json b/scripts/us_census/pep/annual_population/validation_config.json index 96ab51bf96..03ec73d603 100644 --- a/scripts/us_census/pep/annual_population/validation_config.json +++ b/scripts/us_census/pep/annual_population/validation_config.json @@ -25,13 +25,25 @@ "input_files": "../../../../output_files/usa_annual_population.csv" } }, + { + "rule_id": "max_date_consistent", + "validator": "MAX_DATE_CONSISTENT", + "scope": { + "variables": { + "dcids": [ + "*" + ] + } + }, + "params": {} + }, { "rule_id": "check_date_freshness", "description": "Verifies that all StatVars meet freshness requirements (within 2 years of current year).", "validator": "SQL_VALIDATOR", "params": { - "query": "SELECT CAST(MIN(MaxDate) AS VARCHAR) AS latest_date FROM stats", - "condition": "latest_date IS NOT NULL AND CAST(SUBSTR(CAST(latest_date AS VARCHAR), 1, 4) AS INT) >= YEAR(CURRENT_DATE) - 2" + "query": "SELECT StatVar, MaxDate FROM stats", + "condition": "MaxDate IS NOT NULL AND CAST(SUBSTR(CAST(MaxDate AS VARCHAR), 1, 4) AS INT) >= YEAR(CURRENT_DATE) - 2" } } ] diff --git a/scripts/us_census/pep/us_pep_sex/manifest.json b/scripts/us_census/pep/us_pep_sex/manifest.json index bd2fa05ca9..ee29b153a4 100644 --- a/scripts/us_census/pep/us_pep_sex/manifest.json +++ b/scripts/us_census/pep/us_pep_sex/manifest.json @@ -11,7 +11,7 @@ "process.py --config_path=unresolved_mcf/us_census/pep/population_estimate_by_sex/skip_url.json" ], "source_files": [ - "gcs_output/us_pep_sex_source_files/*", + "input_files/*", "golden_data/*.csv" ], "import_inputs": [ diff --git a/scripts/us_census/pep/us_pep_sex/process.py b/scripts/us_census/pep/us_pep_sex/process.py index f5d31a8882..6f68babb07 100644 --- a/scripts/us_census/pep/us_pep_sex/process.py +++ b/scripts/us_census/pep/us_pep_sex/process.py @@ -353,7 +353,7 @@ def _national_latest(file_path: str) -> pd.DataFrame: 2: 'Count_Person_Female' }}) column_list = [] - for year in range(2021, 2030): + for year in range(2021, 2031): column_name = f'POPESTIMATE{year}' if column_name in df.columns: df.rename(columns={column_name: str(year)}, inplace=True) @@ -600,7 +600,7 @@ def _state_latest(file_path: str) -> pd.DataFrame: df = pd.read_excel(file_path, skiprows=5, skipfooter=7, header=None) current_year = int(max_year) + 1 for year in range(2021, current_year): - if current_year <= 2030: + if current_year <= 2031: base_columns.append(f'July{year}Total') base_columns.append(f'July{year}Male') base_columns.append(f'July{year}Female') @@ -616,7 +616,7 @@ def _state_latest(file_path: str) -> pd.DataFrame: df.insert(1, 'geo_ID', 'geoId/' + geoid) processed_dfs = [] # List to hold yearly DataFrames - for year in range(2020, 2030): + for year in range(2020, 2031): # Generate column names dynamically male_col = f'July{year}Male' female_col = f'July{year}Female' @@ -917,7 +917,8 @@ def _county_latest(file_path: str) -> pd.DataFrame: 8: '2026', 9: '2027', 10: '2028', - 11: '2029' + 11: '2029', + 12: '2030' } }) @@ -1018,6 +1019,60 @@ def process(self): processed_count = 0 total_files_to_process = len(ip_files) logging.info(f"No of files to be processed {len(ip_files)}") + # Define the base mappings for fixed years (e.g., 2023) + file_to_function_mapping = { + "pe-11-1": _national_1900_1979, + "us-est90int": _national_1990_2000, + "us-est90int-": _national_1990_2000, + "us-est00int": _national_2000_2010, + "us-est00int-": _national_2000_2010, + "nc-est2020-agesex-": _national_2010_2020, + "nc-est2020-agesex-r": _national_2010_2020, + "pe": _state_1970_1980, + "stiag": _state_1980_1990, + "st-est00int-02-": _state_2000_2010, + "st-est00int-02": _state_2000_2010, + "SC-EST2020-AGESEX": _state_2010_2020, + "SC-EST2020-AGESEX-": _state_2010_2020, + "co-asr-1": _county_1970_1980, + "pe-02-1": _county_1980_1990, + "stch-icen1": _county_1990_2000, + "co-est00int-agesex-": _county_2000_2010, + "co-est00int-agesex-5": _county_2000_2010, + "CC-EST2020-AGESEX-": _county_2010_2020, + "CC-EST2020-AGESEX-A": _county_2010_2020 + } + + # Iterate from 2023 to 2030 and add mappings for years dynamically + for file_year in range(2023, 2031): + # For the 'national', 'state', and 'county' entries, use the same method for years 2023 to 2030 + file_to_function_mapping[ + f"nc-est{file_year}-agesex-"] = _national_latest + file_to_function_mapping[ + f"nc-est{file_year}-agesex-r"] = _national_latest + + file_to_function_mapping[ + f"sc-est{file_year}-syasex-2"] = _state_latest + file_to_function_mapping[ + f"sc-est{file_year}-syasex-3"] = _state_latest + file_to_function_mapping[ + f"sc-est{file_year}-syasex-4"] = _state_latest + file_to_function_mapping[ + f"sc-est{file_year}-syasex-5"] = _state_latest + file_to_function_mapping[ + f"sc-est{file_year}-syasex-0"] = _state_latest + file_to_function_mapping[ + f"sc-est{file_year}-syasex-1"] = _state_latest + file_to_function_mapping[ + f"sc-est{file_year}-syasex-"] = _state_latest + file_to_function_mapping[ + f"sc-est{file_year}-agesex-"] = _state_latest + + file_to_function_mapping[ + f"cc-est{file_year}-agesex-"] = _county_latest + file_to_function_mapping[ + f"cc-est{file_year}-agesex-a"] = _county_latest + for file_path in ip_files: logging.info(f"Processing the file:{file_path}") processed_count += 1 @@ -1027,60 +1082,6 @@ def process(self): file_name = file_path.split("/")[-1][:-7] - # Define the base mappings for fixed years (e.g., 2023) - file_to_function_mapping = { - "pe-11-1": _national_1900_1979, - "us-est90int": _national_1990_2000, - "us-est90int-": _national_1990_2000, - "us-est00int": _national_2000_2010, - "us-est00int-": _national_2000_2010, - "nc-est2020-agesex-": _national_2010_2020, - "nc-est2020-agesex-r": _national_2010_2020, - "pe": _state_1970_1980, - "stiag": _state_1980_1990, - "st-est00int-02-": _state_2000_2010, - "st-est00int-02": _state_2000_2010, - "SC-EST2020-AGESEX": _state_2010_2020, - "SC-EST2020-AGESEX-": _state_2010_2020, - "co-asr-1": _county_1970_1980, - "pe-02-1": _county_1980_1990, - "stch-icen1": _county_1990_2000, - "co-est00int-agesex-": _county_2000_2010, - "co-est00int-agesex-5": _county_2000_2010, - "CC-EST2020-AGESEX-": _county_2010_2020, - "CC-EST2020-AGESEX-A": _county_2010_2020 - } - - # Iterate from 2023 to 2029 and add mappings for years dynamically - for file_year in range(2023, 2030): - # For the 'national', 'state', and 'county' entries, use the same method for years 2023 to 2029 - file_to_function_mapping[ - f"nc-est{file_year}-agesex-"] = _national_latest - file_to_function_mapping[ - f"nc-est{file_year}-agesex-r"] = _national_latest - - file_to_function_mapping[ - f"sc-est{file_year}-syasex-2"] = _state_latest - file_to_function_mapping[ - f"sc-est{file_year}-syasex-3"] = _state_latest - file_to_function_mapping[ - f"sc-est{file_year}-syasex-4"] = _state_latest - file_to_function_mapping[ - f"sc-est{file_year}-syasex-5"] = _state_latest - file_to_function_mapping[ - f"sc-est{file_year}-syasex-0"] = _state_latest - file_to_function_mapping[ - f"sc-est{file_year}-syasex-1"] = _state_latest - file_to_function_mapping[ - f"sc-est{file_year}-syasex-"] = _state_latest - file_to_function_mapping[ - f"sc-est{file_year}-agesex-"] = _state_latest - - file_to_function_mapping[ - f"cc-est{file_year}-agesex-"] = _county_latest - file_to_function_mapping[ - f"cc-est{file_year}-agesex-a"] = _county_latest - df = file_to_function_mapping[file_name](file_path) if not df.empty: @@ -1138,7 +1139,7 @@ def _create_retry_session() -> requests.Session: retries = Retry( total=5, backoff_factor=2, - status_forcelist=[429, 500, 502, 503, 504, 520], + status_forcelist=[408, 429, 500, 502, 503, 504, 520], allowed_methods=["HEAD", "GET"], respect_retry_after_header=True, ) @@ -1148,23 +1149,49 @@ def _create_retry_session() -> requests.Session: return session +@retry(tries=3, delay=2, backoff=2, exceptions=(ValueError,)) +def _head_with_html_retry(session: requests.Session, + url: str, + timeout: int = 15) -> requests.Response: + """Performs a HEAD request and retries if HTTP 200 returns an HTML error page.""" + response = session.head(url, allow_redirects=True, timeout=timeout) + content_type = response.headers.get('Content-Type', '') + if response.status_code == 200 and 'html' in content_type.lower(): + raise ValueError(f"Server returned HTTP 200 HTML response for {url}") + return response + + def is_valid_url(session: requests.Session, url: str) -> bool: try: with session.get(url, stream=True, timeout=20) as response: if response.status_code != 200: return False content_type = response.headers.get("Content-Type", "") - if "text/html" in content_type.lower(): - # Might be an error page disguised as 200 - chunk = next(response.iter_content(chunk_size=1024), b"") - if b"error" in chunk.lower() or len(chunk) < 500: - return False + if "html" in content_type.lower(): + return False return True except Exception as e: logging.warning(f"Error checking URL: {url} - {e}") return False +_BASELINE_YEAR = 2025 + + +def _retain_cached_if_exists(url_to_check: str): + """If a cached file exists in persistent storage for url_to_check, retain it in _FILES_TO_DOWNLOAD.""" + file_name = url_to_check.split('/')[-1] + cached_file_path = os.path.join(_GCS_FOLDER_PERSISTENT_PATH, file_name) + if os.path.isfile(cached_file_path): + logging.warning( + f"HEAD probe failed for {url_to_check}, but cached copy exists at {cached_file_path}; retaining in download list." + ) + _FILES_TO_DOWNLOAD.append({ + "download_path": url_to_check, + "is_historical": False + }) + + def add_future_year_urls(): """ This method adds the future year urls that has to be downloaded @@ -1198,6 +1225,8 @@ def add_future_year_urls(): session, url["download_path"])) # suspicious, but passes check ) ] + for item in _FILES_TO_DOWNLOAD: + item["is_historical"] = True logging.info( f"Historical urls fetched from input_json: {_FILES_TO_DOWNLOAD}") @@ -1212,16 +1241,16 @@ def add_future_year_urls(): # A set to track downloaded URLs for unique {YEAR} and URLs without {i} downloaded_year_urls = set() + selected_year = None - # Loop through years in reverse order from 2030 to 2023 - for future_year in range(2030, 2022, -1): # From 2030 to 2023 - + # Loop through years in reverse order from 2030 down to _BASELINE_YEAR + for future_year in range(2030, _BASELINE_YEAR - 1, -1): # We check the National CSV first. If it's 404, the whole year is skipped. gatekeeper_url = urls_to_scan[0].format(YEAR=future_year) try: - response = session.head(gatekeeper_url, - allow_redirects=True, - timeout=15) + response = _head_with_html_retry(session, + gatekeeper_url, + timeout=15) time.sleep(0.2) content_type = response.headers.get('Content-Type', '') if response.status_code != 200 or 'html' in content_type.lower(): @@ -1229,69 +1258,83 @@ def add_future_year_urls(): f"Skipping year {future_year}: National file not found or is HTML (status code: {response.status_code})." ) continue - except requests.exceptions.RequestException as e: + selected_year = future_year + break + except (requests.exceptions.RequestException, ValueError) as e: logging.warning( f"Skipping year {future_year} due to an error checking the gatekeeper URL: {e}" ) continue - YEAR = future_year - # Loop through URLs - for url in urls_to_scan: - if "{i}" in url: # This URL contains the {i} variable, so we loop through i from 01 to 56 - for i in range(1, 57): # Loop i from 01 to 56 - formatted_i = f"{i:02}" # Ensure i is always 2 digits (01, 02, ..., 56) - url_to_check = url.format(YEAR=YEAR, i=formatted_i) - logging.info(f"checking url: {url_to_check}") - - try: - # HEAD calls only fetch headers and should complete quickly. - check_url = session.head(url_to_check, - allow_redirects=True, - timeout=15) - time.sleep(0.1) - content_type = check_url.headers.get('Content-Type', '') - if check_url.status_code == 200 and 'html' not in content_type.lower( - ): - _FILES_TO_DOWNLOAD.append( - {"download_path": url_to_check}) - - except requests.exceptions.RequestException as e: - logging.error( - f"URL is not accessible {url_to_check} due to {e}") - - else: # This URL does not contain {i}, so we only need to process it once per year - url_to_check = url.format(YEAR=YEAR) + if selected_year is None: + logging.warning( + f"No gatekeeper URL confirmed via HEAD probe; falling back to baseline year {_BASELINE_YEAR}." + ) + selected_year = _BASELINE_YEAR + + YEAR = selected_year + # Loop through URLs for the selected vintage year + for url in urls_to_scan: + if "{i}" in url: # This URL contains the {i} variable, so we loop through i from 01 to 56 + for i in range(1, 57): # Loop i from 01 to 56 + formatted_i = f"{i:02}" # Ensure i is always 2 digits (01, 02, ..., 56) + url_to_check = url.format(YEAR=YEAR, i=formatted_i) logging.info(f"checking url: {url_to_check}") - # If the URL has already been processed for this year, skip it - if url_to_check in downloaded_year_urls: - continue # Skip this URL if it's already processed try: # HEAD calls only fetch headers and should complete quickly. - check_url = session.head(url_to_check, - allow_redirects=True, - timeout=15) + check_url = _head_with_html_retry(session, + url_to_check, + timeout=15) time.sleep(0.1) content_type = check_url.headers.get('Content-Type', '') if check_url.status_code == 200 and 'html' not in content_type.lower( ): - _FILES_TO_DOWNLOAD.append( - {"download_path": url_to_check}) - downloaded_year_urls.add( - url_to_check) # Mark this URL as processed - + _FILES_TO_DOWNLOAD.append({ + "download_path": url_to_check, + "is_historical": False + }) else: - logging.error( - f"URL returned status code {check_url.status_code}: {url_to_check}" - ) + _retain_cached_if_exists(url_to_check) - except requests.exceptions.RequestException as e: + except (requests.exceptions.RequestException, ValueError) as e: logging.error( f"URL is not accessible {url_to_check} due to {e}") + _retain_cached_if_exists(url_to_check) + + else: # This URL does not contain {i}, so we only need to process it once per year + url_to_check = url.format(YEAR=YEAR) + logging.info(f"checking url: {url_to_check}") + # If the URL has already been processed for this year, skip it + if url_to_check in downloaded_year_urls: + continue # Skip this URL if it's already processed + + try: + # HEAD calls only fetch headers and should complete quickly. + check_url = _head_with_html_retry(session, + url_to_check, + timeout=15) + time.sleep(0.1) + content_type = check_url.headers.get('Content-Type', '') + if check_url.status_code == 200 and 'html' not in content_type.lower( + ): + _FILES_TO_DOWNLOAD.append({ + "download_path": url_to_check, + "is_historical": False + }) + downloaded_year_urls.add( + url_to_check) # Mark this URL as processed + + else: + logging.error( + f"URL returned status code {check_url.status_code}: {url_to_check}" + ) + _retain_cached_if_exists(url_to_check) - # Terminate year scan once latest confirmed vintage is processed - break + except (requests.exceptions.RequestException, ValueError) as e: + logging.error( + f"URL is not accessible {url_to_check} due to {e}") + _retain_cached_if_exists(url_to_check) def _get_target_file_name(file_to_download: dict) -> str: @@ -1320,12 +1363,9 @@ def cleanup(): os.remove(file_path) -@retry(tries=5, - delay=3, - backoff=2, - exceptions=(requests.RequestException, ValueError)) +@retry(tries=3, delay=3, backoff=2, exceptions=(ValueError,)) def _download_single_file(session: requests.Session, url: str) -> str: - """Downloads a single URL to a temporary file, retrying on HTTP or HTML error responses.""" + """Downloads a single URL to a temporary file, retrying on HTML error responses.""" with session.get(url, stream=True, timeout=120) as response: response.raise_for_status() content_type = response.headers.get('Content-Type', '') @@ -1349,10 +1389,6 @@ def _download_single_file(session: requests.Session, url: str) -> str: raise -@retry(tries=3, - delay=2, - backoff=2, - exceptions=(requests.RequestException, Exception)) def download_files(): """ Download files from URLs listed in _FILES_TO_DOWNLOAD. @@ -1373,10 +1409,19 @@ def download_files(): file_name_to_save = _get_target_file_name(file_to_download) cached_file_path = os.path.join(_GCS_FOLDER_PERSISTENT_PATH, file_name_to_save) + is_historical = file_to_download.get('is_historical', False) - # Reuse cached file if it exists and is within TTL + # Reuse cached file if it exists (historical files never expire; dynamic files use _TTL_DAYS) if file_name_to_save in downloaded_files and os.path.isfile( cached_file_path): + if is_historical: + logging.info( + f"Skipping download, using immutable historical cached file: {file_name_to_save}" + ) + shutil.copy(cached_file_path, + os.path.join(_INPUT_FILE_PATH, file_name_to_save)) + file_to_download['is_downloaded'] = True + continue file_age = (time.time() - os.path.getmtime(cached_file_path)) / (24 * 3600) if file_age <= _TTL_DAYS: @@ -1417,6 +1462,7 @@ def download_files(): logging.warning( f"Failed to refresh {url} after retries ({e}); falling back to cached file: {file_name_to_save}" ) + os.utime(cached_file_path, None) shutil.copy(cached_file_path, os.path.join(_INPUT_FILE_PATH, file_name_to_save)) file_to_download['is_downloaded'] = True diff --git a/scripts/us_census/pep/us_pep_sex/process_test.py b/scripts/us_census/pep/us_pep_sex/process_test.py index 2dcd71f0b3..780db35945 100644 --- a/scripts/us_census/pep/us_pep_sex/process_test.py +++ b/scripts/us_census/pep/us_pep_sex/process_test.py @@ -19,8 +19,6 @@ import unittest import sys import tempfile -import json -import shutil from unittest.mock import MagicMock, patch import numpy as np import pandas as pd @@ -38,9 +36,7 @@ PopulationEstimateBySex, _state_latest, _download_single_file, - _create_retry_session, - add_future_year_urls, - download_files, + is_valid_url, ) import us_census.pep.us_pep_sex.process as sex_proc # pylint: enable=wrong-import-position @@ -148,12 +144,12 @@ def test_state_latest_boundary_2029(self): class TestDynamicFunctionMapping(unittest.TestCase): - """Tests dynamic registration of file-to-function mappings up to year 2029.""" + """Tests dynamic registration of file-to-function mappings up to year 2030.""" - def test_file_to_function_mapping_years_2023_to_2029(self): - """Tests that national, state, and county future year files (e.g. 2024, 2029) are dispatched correctly.""" + def test_file_to_function_mapping_years_2023_to_2030(self): + """Tests that national, state, and county future year files (e.g. 2024, 2029, 2030) are dispatched correctly.""" with tempfile.TemporaryDirectory() as tmp_dir: - for yr in [2024, 2029]: + for yr in [2024, 2029, 2030]: open(os.path.join(tmp_dir, f"nc-est{yr}-agesex-res.csv"), 'w').close() open(os.path.join(tmp_dir, f"sc-est{yr}-syasex-01.xlsx"), @@ -180,14 +176,26 @@ def test_file_to_function_mapping_years_2023_to_2029(self): os.path.join(tmp_dir, 'out.mcf'), os.path.join(tmp_dir, 'out.tmcf')) loader.process() - self.assertEqual(mock_nat.call_count, 2) - self.assertEqual(mock_st.call_count, 2) - self.assertEqual(mock_co.call_count, 2) + self.assertEqual(mock_nat.call_count, 3) + self.assertEqual(mock_st.call_count, 3) + self.assertEqual(mock_co.call_count, 3) class TestDownloadAndCacheLifecycle(unittest.TestCase): """Tests for streaming downloads, temp file cleanup, and cache fallback.""" + def test_is_valid_url_rejects_html_regardless_of_size(self): + """Tests that is_valid_url returns False immediately for any HTML content-type.""" + session = MagicMock() + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.headers = {'Content-Type': 'text/html; charset=UTF-8'} + mock_resp.iter_content.return_value = [b'' + b'a' * 1000] + mock_resp.__enter__.return_value = mock_resp + session.get.return_value = mock_resp + + self.assertFalse(is_valid_url(session, 'https://census.gov/test.csv')) + def test_download_single_file_success(self): """Tests that _download_single_file streams chunks to temp file.""" session = MagicMock() @@ -249,48 +257,66 @@ def failing_iter(*args, **kwargs): self.assertEqual(len(created_files), 1) self.assertFalse(os.path.exists(created_files[0])) - def test_download_files_cache_fallback(self): - """Tests that download_files falls back to cached persistent file if network download fails.""" + def test_download_files_cache_fallback_and_historical_ttl_exemption(self): + """Tests that historical files bypass TTL and expired dynamic files update mtime on fallback.""" with tempfile.TemporaryDirectory() as tmp_dir: gcs_persistent = os.path.join(tmp_dir, 'gcs_persistent') input_dir = os.path.join(tmp_dir, 'input_files') os.makedirs(gcs_persistent) os.makedirs(input_dir) - cached_file = os.path.join(gcs_persistent, - 'nc-est2024-agesex-res.csv') - with open(cached_file, 'w') as f: - f.write('cached_content') - - # Force file to appear expired to trigger download attempt - os.utime(cached_file, (0, 0)) + hist_cached = os.path.join(gcs_persistent, 'pe-19.csv') + with open(hist_cached, 'w') as f: + f.write('historical_content') + os.utime(hist_cached, (0, 0)) - files_to_download = [{ - 'download_path': 'https://census.gov/nc-est2024-agesex-res.csv' - }] + dyn_cached = os.path.join(gcs_persistent, + 'nc-est2025-agesex-res.csv') + with open(dyn_cached, 'w') as f: + f.write('cached_content') + os.utime(dyn_cached, (0, 0)) + + files_to_download = [ + { + 'download_path': 'https://census.gov/pe-19.csv', + 'is_historical': True, + }, + { + 'download_path': + 'https://census.gov/nc-est2025-agesex-res.csv', + 'is_historical': False, + }, + ] with patch.object(sex_proc, '_GCS_FOLDER_PERSISTENT_PATH', gcs_persistent), \ patch.object(sex_proc, '_INPUT_FILE_PATH', input_dir), \ patch.object(sex_proc, '_FILES_TO_DOWNLOAD', files_to_download), \ - patch.object(sex_proc, '_download_single_file', side_effect=IOError('Network error')), \ + patch.object(sex_proc, '_download_single_file', side_effect=IOError('Network error')) as mock_dl, \ patch('time.sleep'): - sex_proc.download_files.__wrapped__() + sex_proc.download_files() + # Only the expired non-historical file should have attempted a network download + self.assertEqual(mock_dl.call_count, 1) - dest_file = os.path.join(input_dir, 'nc-est2024-agesex-res.csv') + dest_hist = os.path.join(input_dir, 'pe-19.csv') + self.assertTrue(os.path.exists(dest_hist)) + dest_file = os.path.join(input_dir, 'nc-est2025-agesex-res.csv') self.assertTrue(os.path.exists(dest_file)) with open(dest_file) as f: self.assertEqual(f.read(), 'cached_content') self.assertTrue(files_to_download[0]['is_downloaded']) + self.assertTrue(files_to_download[1]['is_downloaded']) + # Verify mtime of fallback cached file was refreshed + self.assertGreater(os.path.getmtime(dyn_cached), 0) - def test_add_future_year_urls_gatekeeper_break(self): - """Tests that add_future_year_urls gatekeeper loop breaks immediately upon finding the first valid year.""" + def test_add_future_year_urls_gatekeeper_break_and_cache_retention(self): + """Tests that add_future_year_urls breaks at first valid year, bounds at baseline, and retains cached files on HEAD failure.""" session = MagicMock() head_calls = [] def mock_head(url, *args, **kwargs): head_calls.append(url) resp = MagicMock() - if '2024' in url: + if '2026' in url and 'syasex-01' not in url: resp.status_code = 200 resp.headers = {'Content-Type': 'text/csv'} else: @@ -300,14 +326,28 @@ def mock_head(url, *args, **kwargs): session.head = mock_head - with patch.object(sex_proc, 'extract_gcs_info', return_value=('bucket', 'path')), \ - patch.object(sex_proc, 'fetch_skip_urls_from_gcs', return_value=set()), \ - patch.object(sex_proc, '_create_retry_session', return_value=session), \ - patch('time.sleep'): - sex_proc.add_future_year_urls() - - self.assertTrue(any('2024' in url for url in head_calls)) - self.assertFalse(any('2023' in url for url in head_calls)) + with tempfile.TemporaryDirectory() as tmp_dir: + # Create a cached file for sc-est2026-syasex-01.xlsx whose HEAD probe will fail + cached_state = os.path.join(tmp_dir, 'sc-est2026-syasex-01.xlsx') + with open(cached_state, 'w') as f: + f.write('cached') + + with patch.object(sex_proc, '_FILES_TO_DOWNLOAD', []), \ + patch.object(sex_proc, '_GCS_FOLDER_PERSISTENT_PATH', tmp_dir), \ + patch.object(sex_proc, 'extract_gcs_info', return_value=('bucket', 'path')), \ + patch.object(sex_proc, 'fetch_skip_urls_from_gcs', return_value=set()), \ + patch.object(sex_proc, '_create_retry_session', return_value=session), \ + patch('time.sleep'): + sex_proc.add_future_year_urls() + downloaded_paths = [ + item['download_path'] + for item in sex_proc._FILES_TO_DOWNLOAD + ] + + self.assertTrue(any('2026' in url for url in head_calls)) + self.assertFalse(any('2025' in url for url in head_calls)) + self.assertTrue( + any('sc-est2026-syasex-01.xlsx' in url for url in downloaded_paths)) if __name__ == "__main__": diff --git a/scripts/us_census/pep/us_pep_sex/validation_config.json b/scripts/us_census/pep/us_pep_sex/validation_config.json index daaaea080f..1739d2566c 100644 --- a/scripts/us_census/pep/us_pep_sex/validation_config.json +++ b/scripts/us_census/pep/us_pep_sex/validation_config.json @@ -25,13 +25,25 @@ "input_files": "../../../../output/population_estimate_sex.csv" } }, + { + "rule_id": "max_date_consistent", + "validator": "MAX_DATE_CONSISTENT", + "scope": { + "variables": { + "dcids": [ + "*" + ] + } + }, + "params": {} + }, { "rule_id": "check_date_freshness", "description": "Verifies that all StatVars meet freshness requirements (within 2 years of current year).", "validator": "SQL_VALIDATOR", "params": { - "query": "SELECT CAST(MIN(MaxDate) AS VARCHAR) AS latest_date FROM stats", - "condition": "latest_date IS NOT NULL AND CAST(SUBSTR(CAST(latest_date AS VARCHAR), 1, 4) AS INT) >= YEAR(CURRENT_DATE) - 2" + "query": "SELECT StatVar, MaxDate FROM stats", + "condition": "MaxDate IS NOT NULL AND CAST(SUBSTR(CAST(MaxDate AS VARCHAR), 1, 4) AS INT) >= YEAR(CURRENT_DATE) - 2" } } ] From d6ac7ca03a368d5a27c9fbc8ff32d37836fe0100 Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Wed, 14 Oct 2026 05:25:28 +0000 Subject: [PATCH 14/15] resolved comments --- .../pep/annual_population/validation_config.json | 12 ------------ .../us_census/pep/us_pep_sex/validation_config.json | 12 ------------ 2 files changed, 24 deletions(-) diff --git a/scripts/us_census/pep/annual_population/validation_config.json b/scripts/us_census/pep/annual_population/validation_config.json index 03ec73d603..a5adce7684 100644 --- a/scripts/us_census/pep/annual_population/validation_config.json +++ b/scripts/us_census/pep/annual_population/validation_config.json @@ -25,18 +25,6 @@ "input_files": "../../../../output_files/usa_annual_population.csv" } }, - { - "rule_id": "max_date_consistent", - "validator": "MAX_DATE_CONSISTENT", - "scope": { - "variables": { - "dcids": [ - "*" - ] - } - }, - "params": {} - }, { "rule_id": "check_date_freshness", "description": "Verifies that all StatVars meet freshness requirements (within 2 years of current year).", diff --git a/scripts/us_census/pep/us_pep_sex/validation_config.json b/scripts/us_census/pep/us_pep_sex/validation_config.json index 1739d2566c..56fa700e5f 100644 --- a/scripts/us_census/pep/us_pep_sex/validation_config.json +++ b/scripts/us_census/pep/us_pep_sex/validation_config.json @@ -25,18 +25,6 @@ "input_files": "../../../../output/population_estimate_sex.csv" } }, - { - "rule_id": "max_date_consistent", - "validator": "MAX_DATE_CONSISTENT", - "scope": { - "variables": { - "dcids": [ - "*" - ] - } - }, - "params": {} - }, { "rule_id": "check_date_freshness", "description": "Verifies that all StatVars meet freshness requirements (within 2 years of current year).", From 44780fa5f806070b4e715e8083e7afb48f48eed1 Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Thu, 15 Oct 2026 17:47:13 +0000 Subject: [PATCH 15/15] fixed lint error --- scripts/us_census/pep/annual_population/preprocess_test.py | 4 ++-- scripts/us_census/pep/us_pep_sex/process_test.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/us_census/pep/annual_population/preprocess_test.py b/scripts/us_census/pep/annual_population/preprocess_test.py index 04eb0d7d86..21adc5994c 100644 --- a/scripts/us_census/pep/annual_population/preprocess_test.py +++ b/scripts/us_census/pep/annual_population/preprocess_test.py @@ -331,8 +331,8 @@ def mock_head(url, *args, **kwargs): # Ensure 2025-baseline templates never probed 2024 self.assertFalse( any('NST-EST2024-POP.xlsx' in url for url in head_calls)) - self.assertTrue(any('NST-EST2025-POP.xlsx' in url - for url in download_urls)) + self.assertTrue( + any('NST-EST2025-POP.xlsx' in url for url in download_urls)) self.assertTrue( any('NST-EST2023-POPCHG2020_2023.csv' in url for url in download_urls)) diff --git a/scripts/us_census/pep/us_pep_sex/process_test.py b/scripts/us_census/pep/us_pep_sex/process_test.py index 780db35945..65460c6689 100644 --- a/scripts/us_census/pep/us_pep_sex/process_test.py +++ b/scripts/us_census/pep/us_pep_sex/process_test.py @@ -284,7 +284,8 @@ def test_download_files_cache_fallback_and_historical_ttl_exemption(self): { 'download_path': 'https://census.gov/nc-est2025-agesex-res.csv', - 'is_historical': False, + 'is_historical': + False, }, ]