From 103c49ed40ba7430bbee84a2ffe2ab945ae9b3b3 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Fri, 11 Sep 2026 14:14:36 +0000 Subject: [PATCH 1/7] Modified preprocess.py script --- .../commerce_ntia/README.md | 3 + .../commerce_ntia/commerce_ntia_test.py | 98 +++++++++++++++++++ .../commerce_ntia/manifest.json | 16 +-- .../commerce_ntia/preprocess.py | 67 ++++++++----- .../commerce_ntia/validation_config.json | 13 +++ 5 files changed, 167 insertions(+), 30 deletions(-) create mode 100644 statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py create mode 100644 statvar_imports/ntia_internet_use_survey/commerce_ntia/validation_config.json diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md b/statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md index eb6e1e8ea1..b2947f2286 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md @@ -19,6 +19,7 @@ python3 stat_var_processor.py --input_data='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/input_files/' --pv_map='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/' --config_file='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/' --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA//' +--output_counters='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA//' ``` #### Download the data: @@ -40,6 +41,7 @@ python3 stat_var_processor.py --pv_map=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_pvmap.csv --config_file=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/output_files/ntia_output +--output_counters=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/counters/ntia_output_counters.csv ``` ``` @@ -48,5 +50,6 @@ python3 stat_var_processor.py --pv_map=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_age_pvmap.csv --config_file=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/output_files/ntia_age_output +--output_counters=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/counters/ntia_age_output_counters.csv ``` diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py b/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py new file mode 100644 index 0000000000..d6210019a6 --- /dev/null +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py @@ -0,0 +1,98 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit and regression tests for commerce_ntia statvar import.""" + +import os +import subprocess +import sys +import tempfile +import unittest + +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +_DATA_DIR = os.path.abspath(os.path.join(_SCRIPT_DIR, '../../../')) +_TOOLS_DIR = os.path.join(_DATA_DIR, 'tools/statvar_importer') +sys.path.insert(0, _TOOLS_DIR) +sys.path.insert(0, os.path.join(_DATA_DIR, 'util')) +from counters import Counters +from mcf_diff import diff_mcf_files +from mcf_file_util import load_mcf_nodes + + +class CommerceNtiaTest(unittest.TestCase): + + def setUp(self): + self.testdata_dir = os.path.join(_SCRIPT_DIR, 'testdata') + self.processor_path = os.path.join(_TOOLS_DIR, 'stat_var_processor.py') + self.pv_map = os.path.join(_SCRIPT_DIR, 'ntia_pvmap.csv') + self.metadata = os.path.join(_SCRIPT_DIR, 'ntia_metadata.csv') + + def test_stat_var_processor_ntia_output(self): + """Tests that stat_var_processor generates expected outputs for ntia-data.csv.""" + with tempfile.TemporaryDirectory() as tmp_dir: + output_path = os.path.join(tmp_dir, 'ntia_output') + cmd = [ + sys.executable, + self.processor_path, + f'--input_data={os.path.join(self.testdata_dir, "ntia-data.csv")}', + f'--pv_map={self.pv_map}', + f'--config_file={self.metadata}', + f'--output_path={output_path}', + ] + res = subprocess.run(cmd, capture_output=True, text=True) + self.assertEqual(res.returncode, 0, + f'Processor failed: {res.stderr}') + + # Verify CSV + gen_csv = os.path.join(tmp_dir, 'ntia_output.csv') + exp_csv = os.path.join(self.testdata_dir, 'ntia_output.csv') + with open(gen_csv, + encoding='utf-8') as g, open(exp_csv, + encoding='utf-8') as e: + self.assertEqual(g.read().strip(), e.read().strip()) + + # Verify TMCF + gen_tmcf = os.path.join(tmp_dir, 'ntia_output.tmcf') + exp_tmcf = os.path.join(self.testdata_dir, 'ntia_output.tmcf') + with open(gen_tmcf, + encoding='utf-8') as g, open(exp_tmcf, + encoding='utf-8') as e: + self.assertEqual(g.read().strip(), e.read().strip()) + + # Verify StatVar MCF + gen_mcf = os.path.join(tmp_dir, 'ntia_output_stat_vars.mcf') + exp_mcf = os.path.join(self.testdata_dir, + 'ntia_output_stat_vars.mcf') + gen_nodes = load_mcf_nodes(gen_mcf) + exp_nodes = load_mcf_nodes(exp_mcf) + filtered_exp_mcf = os.path.join(tmp_dir, 'filtered_exp.mcf') + filtered_nodes = { + dcid: exp_nodes[dcid] + for dcid in gen_nodes + if dcid in exp_nodes + } + with open(filtered_exp_mcf, 'w', encoding='utf-8') as f: + for dcid, pvs in filtered_nodes.items(): + f.write(f'Node: {dcid}\n') + for p, v in pvs.items(): + if p != 'Node': + f.write(f'{p}: {v}\n') + f.write('\n') + counters = Counters() + diff = diff_mcf_files(gen_mcf, filtered_exp_mcf, + {'show_diff_nodes_only': True}, counters) + self.assertEqual(len(diff), 0, f'MCF diff found: {diff}') + + +if __name__ == '__main__': + unittest.main() diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json b/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json index 2906cb50c3..ca3902d612 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json @@ -9,23 +9,27 @@ "provenance_description": "NTIA programs and policymaking focus largely on expanding broadband Internet access and adoption in America, expanding the use of spectrum by all users.", "scripts": [ "preprocess.py", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ntia-data.csv --pv_map=ntia_pvmap.csv --config_file=ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ntia_output", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ntia-data-age-only.csv --pv_map=ntia_age_pvmap.csv --config_file=ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ntia_age_output" + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ntia-data.csv --pv_map=ntia_pvmap.csv --config_file=ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ntia_output --output_counters=counters/ntia_output_counters.csv", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ntia-data-age-only.csv --pv_map=ntia_age_pvmap.csv --config_file=ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ntia_age_output --output_counters=counters/ntia_age_output_counters.csv" ], "source_files": [ - "input_files/ntia-analyze-table.csv" + "input_files/ntia-analyze-table.csv", + "counters/*.csv" ], "import_inputs": [ { "template_mcf": "output_files/ntia_output.tmcf", - "cleaned_csv": "output_files/ntia_output.csv" + "cleaned_csv": "output_files/ntia_output.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/ntia_age_output.tmcf", - "cleaned_csv": "output_files/ntia_age_output.csv" + "cleaned_csv": "output_files/ntia_age_output.csv", + "node_mcf": "output_files/*.mcf" } ], - "cron_schedule": "0 06 * * 5" + "cron_schedule": "0 06 * * 5", + "validation_config_file": "validation_config.json" } ] } \ No newline at end of file diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py b/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py index 9bcf786a14..d5ea296137 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os, sys +import os +import sys import pandas as pd from absl import app, logging -from pathlib import Path import config script_dir = os.path.dirname(os.path.abspath(__file__)) @@ -27,15 +27,21 @@ Commerce_NTIA_URL = config.Commerce_NTIA_URL INPUT_DIR = os.path.join(script_dir, "input_files") -Path(INPUT_DIR).mkdir(parents=True, exist_ok=True) - COMMON_COLUMNS = ["dataset", "variable", "description", "universe"] -AGE_COLUMNS = ["age314Count", "age1524Count", "age2544Count", "age4564Count", "age65pCount"] +AGE_COLUMNS = [ + "age314Count", "age1524Count", "age2544Count", "age4564Count", "age65pCount" +] INPUT_FILE = os.path.join(INPUT_DIR, "ntia-analyze-table.csv") INPUT_FILE_1 = os.path.join(INPUT_DIR, "ntia-data-age-only.csv") INPUT_FILE_2 = os.path.join(INPUT_DIR, "ntia-data.csv") +HEADERS = { + 'User-Agent': ('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'), + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', +} + def move_column_left(df, column_to_move, target_column): """Moves the universe column to the left of variable column.""" @@ -50,45 +56,58 @@ def move_column_left(df, column_to_move, target_column): def preprocess_data(): try: + os.makedirs(INPUT_DIR, exist_ok=True) org_df = pd.read_csv(INPUT_FILE) - + df1 = org_df[COMMON_COLUMNS + AGE_COLUMNS].copy() df1['universeAgeResol'] = df1['universe'].apply( - lambda x: 'CivilPerson' if x == 'isPerson' else ('Adult' if x == 'isAdult' else None) - ) + lambda x: 'CivilPerson' + if x == 'isPerson' else ('Adult' if x == 'isAdult' else None)) df1['variableAgeResol'] = df1['variable'].apply( - lambda x: 'CivilPerson' if x == 'isPerson' else ('Adult' if x == 'isAdult' else None) - ) + lambda x: 'CivilPerson' + if x == 'isPerson' else ('Adult' if x == 'isAdult' else None)) df1_moved = move_column_left(df1, 'universe', 'variable') df1_moved.to_csv(INPUT_FILE_1, index=False) - df2_cols_to_keep = [col for col in org_df.columns if not col.startswith('age')] + df2_cols_to_keep = [ + col for col in org_df.columns if not col.startswith('age') + ] df2 = org_df[df2_cols_to_keep].copy() df2['universeAgeResol'] = df2['universe'].apply( - lambda x: 'CivilPerson' if x == 'isPerson' else ('Adult' if x == 'isAdult' else None) - ) + lambda x: 'CivilPerson' + if x == 'isPerson' else ('Adult' if x == 'isAdult' else None)) df2['variableAgeResol'] = df2['variable'].apply( - lambda x: 'CivilPerson' if x == 'isPerson' else ('Adult' if x == 'isAdult' else None) - ) + lambda x: 'CivilPerson' + if x == 'isPerson' else ('Adult' if x == 'isAdult' else None)) df2_moved = move_column_left(df2, 'universe', 'variable') df2_moved.to_csv(INPUT_FILE_2, index=False) except Exception as e: - logging.fatal(f"An error occurred while preprocessing the input data: {e}") - return None + logging.fatal( + f"An error occurred while preprocessing the input data: {e}") + sys.exit(1) + def main(argv): try: - download_file(url=Commerce_NTIA_URL, - output_folder=INPUT_DIR, - unzip=False, - headers= None, - tries= 3, - delay= 5, - backoff= 2) + success = download_file( + url=Commerce_NTIA_URL, + output_folder=INPUT_DIR, + unzip=False, + headers=HEADERS, + tries=3, + delay=5, + backoff=2, + ) + if not success or not os.path.exists(INPUT_FILE): + logging.fatal("Failed to download Commerce_NTIA file.") + sys.exit(1) except Exception as e: logging.fatal(f"Failed to download Commerce_NTIA file: {e}") + sys.exit(1) + preprocess_data() + if __name__ == "__main__": app.run(main) diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/validation_config.json b/statvar_imports/ntia_internet_use_survey/commerce_ntia/validation_config.json new file mode 100644 index 0000000000..7249b48bfe --- /dev/null +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/validation_config.json @@ -0,0 +1,13 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted records for the entire import is within threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.1 + } + } + ] +} \ No newline at end of file From f7d73ada0aa08afdeb45bcd601d87c21e4a6a40f Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Fri, 11 Sep 2026 16:39:38 +0000 Subject: [PATCH 2/7] Made changes suggested by CRA --- .../commerce_ntia/README.md | 32 +-- .../commerce_ntia/commerce_ntia_test.py | 183 +++++++++++------- .../commerce_ntia/preprocess.py | 12 +- .../commerce_ntia/validation_config.json | 9 + 4 files changed, 148 insertions(+), 88 deletions(-) diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md b/statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md index b2947f2286..0696662625 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md @@ -5,8 +5,8 @@ - NTIA programs and policymaking focus largely on expanding broadband Internet access and adoption in America, expanding the use of spectrum by all users. - how to download data: - To download and process the data, you'll need to run the provided preprocess script, `preprocess.py`. This script will automatically create an "input_files" folder where you should place the file to be processed. - By using this script, we are creating two more columns in the input files such as 'universeAgeResol', 'variableAgeResol'. This columns are created based on the universe and variable columns in the existing data. + To download and process the data, you'll need to run the provided preprocess script, `preprocess.py`. This script will automatically create an "input_files" folder and download the file to be processed. + This script organizes the data and splits it into general survey data and age-breakdown data. - type of place: Demographics. @@ -16,10 +16,10 @@ ``` python3 stat_var_processor.py ---input_data='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/input_files/' ---pv_map='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/' --config_file='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/' --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf ---output_path='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA//' ---output_counters='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA//' +--input_data='../../statvar_imports/ntia_internet_use_survey/commerce_ntia/input_files/' +--pv_map='../../statvar_imports/ntia_internet_use_survey/commerce_ntia/' --config_file='../../statvar_imports/ntia_internet_use_survey/commerce_ntia/' --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf +--output_path='../../statvar_imports/ntia_internet_use_survey/commerce_ntia//' +--output_counters='../../statvar_imports/ntia_internet_use_survey/commerce_ntia//' ``` #### Download the data: @@ -37,19 +37,19 @@ Execute the script inside the folder `/data/tools/statvar_importer/` ``` python3 stat_var_processor.py ---input_data=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/input_files/ntia-data.csv ---pv_map=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_pvmap.csv ---config_file=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf ---output_path=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/output_files/ntia_output ---output_counters=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/counters/ntia_output_counters.csv +--input_data=../../statvar_imports/ntia_internet_use_survey/commerce_ntia/input_files/ntia-data.csv +--pv_map=../../statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_pvmap.csv +--config_file=../../statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf +--output_path=../../statvar_imports/ntia_internet_use_survey/commerce_ntia/output_files/ntia_output +--output_counters=../../statvar_imports/ntia_internet_use_survey/commerce_ntia/counters/ntia_output_counters.csv ``` ``` python3 stat_var_processor.py ---input_data=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/input_files/ntia-data-age-only.csv ---pv_map=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_age_pvmap.csv ---config_file=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf ---output_path=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/output_files/ntia_age_output ---output_counters=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/counters/ntia_age_output_counters.csv +--input_data=../../statvar_imports/ntia_internet_use_survey/commerce_ntia/input_files/ntia-data-age-only.csv +--pv_map=../../statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_age_pvmap.csv +--config_file=../../statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf +--output_path=../../statvar_imports/ntia_internet_use_survey/commerce_ntia/output_files/ntia_age_output +--output_counters=../../statvar_imports/ntia_internet_use_survey/commerce_ntia/counters/ntia_age_output_counters.csv ``` diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py b/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py index d6210019a6..61eeac6ee8 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py @@ -11,87 +11,134 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Unit and regression tests for commerce_ntia statvar import.""" +"""Hermetic unit tests for commerce_ntia preprocess module.""" import os -import subprocess import sys import tempfile import unittest +from unittest import mock +import pandas as pd _SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -_DATA_DIR = os.path.abspath(os.path.join(_SCRIPT_DIR, '../../../')) -_TOOLS_DIR = os.path.join(_DATA_DIR, 'tools/statvar_importer') -sys.path.insert(0, _TOOLS_DIR) -sys.path.insert(0, os.path.join(_DATA_DIR, 'util')) -from counters import Counters -from mcf_diff import diff_mcf_files -from mcf_file_util import load_mcf_nodes +sys.path.insert(0, _SCRIPT_DIR) +import preprocess -class CommerceNtiaTest(unittest.TestCase): +class PreprocessTest(unittest.TestCase): - def setUp(self): - self.testdata_dir = os.path.join(_SCRIPT_DIR, 'testdata') - self.processor_path = os.path.join(_TOOLS_DIR, 'stat_var_processor.py') - self.pv_map = os.path.join(_SCRIPT_DIR, 'ntia_pvmap.csv') - self.metadata = os.path.join(_SCRIPT_DIR, 'ntia_metadata.csv') + def test_move_column_left_success(self): + """Tests that move_column_left places column immediately left of target.""" + df = pd.DataFrame({'a': [1], 'b': [2], 'c': [3], 'd': [4]}) + result = preprocess.move_column_left(df, 'd', 'b') + self.assertEqual(list(result.columns), ['a', 'd', 'b', 'c']) - def test_stat_var_processor_ntia_output(self): - """Tests that stat_var_processor generates expected outputs for ntia-data.csv.""" + def test_move_column_left_missing_cols(self): + """Tests that move_column_left returns original df if columns are not present.""" + df = pd.DataFrame({'a': [1], 'b': [2]}) + result = preprocess.move_column_left(df, 'missing', 'b') + self.assertEqual(list(result.columns), ['a', 'b']) + + def test_preprocess_data(self): + """Tests data preprocessing and splitting into age-only and general survey CSVs.""" with tempfile.TemporaryDirectory() as tmp_dir: - output_path = os.path.join(tmp_dir, 'ntia_output') - cmd = [ - sys.executable, - self.processor_path, - f'--input_data={os.path.join(self.testdata_dir, "ntia-data.csv")}', - f'--pv_map={self.pv_map}', - f'--config_file={self.metadata}', - f'--output_path={output_path}', - ] - res = subprocess.run(cmd, capture_output=True, text=True) - self.assertEqual(res.returncode, 0, - f'Processor failed: {res.stderr}') - - # Verify CSV - gen_csv = os.path.join(tmp_dir, 'ntia_output.csv') - exp_csv = os.path.join(self.testdata_dir, 'ntia_output.csv') - with open(gen_csv, - encoding='utf-8') as g, open(exp_csv, - encoding='utf-8') as e: - self.assertEqual(g.read().strip(), e.read().strip()) - - # Verify TMCF - gen_tmcf = os.path.join(tmp_dir, 'ntia_output.tmcf') - exp_tmcf = os.path.join(self.testdata_dir, 'ntia_output.tmcf') - with open(gen_tmcf, - encoding='utf-8') as g, open(exp_tmcf, - encoding='utf-8') as e: - self.assertEqual(g.read().strip(), e.read().strip()) - - # Verify StatVar MCF - gen_mcf = os.path.join(tmp_dir, 'ntia_output_stat_vars.mcf') - exp_mcf = os.path.join(self.testdata_dir, - 'ntia_output_stat_vars.mcf') - gen_nodes = load_mcf_nodes(gen_mcf) - exp_nodes = load_mcf_nodes(exp_mcf) - filtered_exp_mcf = os.path.join(tmp_dir, 'filtered_exp.mcf') - filtered_nodes = { - dcid: exp_nodes[dcid] - for dcid in gen_nodes - if dcid in exp_nodes + input_file = os.path.join(tmp_dir, 'ntia-analyze-table.csv') + output_age = os.path.join(tmp_dir, 'ntia-data-age-only.csv') + output_data = os.path.join(tmp_dir, 'ntia-data.csv') + + raw_data = { + 'dataset': ['Nov 2023', 'Nov 2023', 'Nov 2023'], + 'variable': ['Streaming', 'Email', 'Broadband'], + 'description': ['Desc 1', 'Desc 2', 'Desc 3'], + 'universe': ['isPerson', 'isAdult', 'isHousehold'], + 'age314Count': [10, 20, 30], + 'age1524Count': [11, 21, 31], + 'age2544Count': [12, 22, 32], + 'age4564Count': [13, 23, 33], + 'age65pCount': [14, 24, 34], + 'totalCount': [100, 200, 300], + 'otherMetric': [1.5, 2.5, 3.5] } - with open(filtered_exp_mcf, 'w', encoding='utf-8') as f: - for dcid, pvs in filtered_nodes.items(): - f.write(f'Node: {dcid}\n') - for p, v in pvs.items(): - if p != 'Node': - f.write(f'{p}: {v}\n') - f.write('\n') - counters = Counters() - diff = diff_mcf_files(gen_mcf, filtered_exp_mcf, - {'show_diff_nodes_only': True}, counters) - self.assertEqual(len(diff), 0, f'MCF diff found: {diff}') + pd.DataFrame(raw_data).to_csv(input_file, index=False) + + with mock.patch.object(preprocess, 'INPUT_DIR', tmp_dir), \ + mock.patch.object(preprocess, 'INPUT_FILE', input_file), \ + mock.patch.object(preprocess, 'INPUT_FILE_1', output_age), \ + mock.patch.object(preprocess, 'INPUT_FILE_2', output_data): + preprocess.preprocess_data() + + self.assertTrue(os.path.exists(output_age)) + self.assertTrue(os.path.exists(output_data)) + + df_age = pd.read_csv(output_age) + cols_age = list(df_age.columns) + self.assertEqual( + cols_age.index('universe') + 1, cols_age.index('variable')) + for age_col in preprocess.AGE_COLUMNS: + self.assertIn(age_col, cols_age) + self.assertNotIn('totalCount', cols_age) + self.assertNotIn('otherMetric', cols_age) + self.assertIn('universeAgeResol', cols_age) + self.assertIn('variableAgeResol', cols_age) + self.assertEqual(df_age.loc[0, 'universeAgeResol'], 'CivilPerson') + self.assertEqual(df_age.loc[1, 'universeAgeResol'], 'Adult') + self.assertTrue(pd.isna(df_age.loc[2, 'universeAgeResol'])) + + df_data = pd.read_csv(output_data) + cols_data = list(df_data.columns) + self.assertEqual( + cols_data.index('universe') + 1, cols_data.index('variable')) + self.assertIn('totalCount', cols_data) + self.assertIn('otherMetric', cols_data) + self.assertIn('universeAgeResol', cols_data) + self.assertIn('variableAgeResol', cols_data) + self.assertEqual(df_data.loc[0, 'universeAgeResol'], 'CivilPerson') + self.assertEqual(df_data.loc[1, 'universeAgeResol'], 'Adult') + self.assertTrue(pd.isna(df_data.loc[2, 'universeAgeResol'])) + for age_col in preprocess.AGE_COLUMNS: + self.assertNotIn(age_col, cols_data) + + @mock.patch('preprocess.preprocess_data') + @mock.patch('preprocess.download_file') + def test_main_download_success(self, mock_download, mock_preprocess): + """Tests that main downloads file and executes preprocess_data on success.""" + mock_download.return_value = True + with mock.patch('os.path.exists', return_value=True): + preprocess.main([]) + mock_download.assert_called_once_with( + url=preprocess.Commerce_NTIA_URL, + output_folder=preprocess.INPUT_DIR, + unzip=False, + headers=preprocess.HEADERS, + tries=3, + delay=5, + backoff=2, + ) + mock_preprocess.assert_called_once() + + @mock.patch('preprocess.preprocess_data') + @mock.patch('preprocess.download_file') + @mock.patch('preprocess.logging.fatal') + def test_main_download_failure(self, mock_fatal, mock_download, + mock_preprocess): + """Tests that main logs fatal error and halts when download returns False.""" + mock_download.return_value = False + preprocess.main([]) + mock_fatal.assert_called_once_with( + "Failed to download Commerce_NTIA file.") + mock_preprocess.assert_not_called() + + @mock.patch('preprocess.preprocess_data') + @mock.patch('preprocess.download_file') + @mock.patch('preprocess.logging.fatal') + def test_main_download_exception(self, mock_fatal, mock_download, + mock_preprocess): + """Tests that main logs fatal error when download raises an exception.""" + mock_download.side_effect = Exception("Connection timeout") + preprocess.main([]) + self.assertTrue(mock_fatal.called) + self.assertIn("Connection timeout", str(mock_fatal.call_args)) + mock_preprocess.assert_not_called() if __name__ == '__main__': diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py b/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py index d5ea296137..6ca9a532be 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py @@ -30,7 +30,8 @@ COMMON_COLUMNS = ["dataset", "variable", "description", "universe"] AGE_COLUMNS = [ - "age314Count", "age1524Count", "age2544Count", "age4564Count", "age65pCount" + "age314Count", "age1524Count", "age2544Count", "age4564Count", + "age65pCount" ] INPUT_FILE = os.path.join(INPUT_DIR, "ntia-analyze-table.csv") INPUT_FILE_1 = os.path.join(INPUT_DIR, "ntia-data-age-only.csv") @@ -39,7 +40,8 @@ HEADERS = { 'User-Agent': ('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'), - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept': + 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', } @@ -59,6 +61,7 @@ def preprocess_data(): os.makedirs(INPUT_DIR, exist_ok=True) org_df = pd.read_csv(INPUT_FILE) + # 1. Process Age-only data df1 = org_df[COMMON_COLUMNS + AGE_COLUMNS].copy() df1['universeAgeResol'] = df1['universe'].apply( lambda x: 'CivilPerson' @@ -69,6 +72,7 @@ def preprocess_data(): df1_moved = move_column_left(df1, 'universe', 'variable') df1_moved.to_csv(INPUT_FILE_1, index=False) + # 2. Process General survey data df2_cols_to_keep = [ col for col in org_df.columns if not col.startswith('age') ] @@ -101,10 +105,10 @@ def main(argv): ) if not success or not os.path.exists(INPUT_FILE): logging.fatal("Failed to download Commerce_NTIA file.") - sys.exit(1) + return except Exception as e: logging.fatal(f"Failed to download Commerce_NTIA file: {e}") - sys.exit(1) + return preprocess_data() diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/validation_config.json b/statvar_imports/ntia_internet_use_survey/commerce_ntia/validation_config.json index 7249b48bfe..c94065b0c2 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/validation_config.json +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/validation_config.json @@ -8,6 +8,15 @@ "params": { "threshold": 0.1 } + }, + { + "rule_id": "check_max_date_freshness", + "description": "Checks that the latest observation date is at least 2023.", + "validator": "SQL_VALIDATOR", + "params": { + "query": "SELECT MAX(MaxDate) AS max_date FROM stats", + "condition": "max_date >= '2023'" + } } ] } \ No newline at end of file From 08d1a1ba04f1190304eeb34c3dcec35a52b9ad54 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Mon, 14 Sep 2026 07:26:32 +0000 Subject: [PATCH 3/7] Made changes suggested by CRA analysis --- .../commerce_ntia/commerce_ntia_test.py | 79 +++++++++++++++++-- .../commerce_ntia/preprocess.py | 19 +++-- 2 files changed, 84 insertions(+), 14 deletions(-) diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py b/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py index 61eeac6ee8..0d4bbba7ad 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py @@ -39,6 +39,12 @@ def test_move_column_left_missing_cols(self): result = preprocess.move_column_left(df, 'missing', 'b') self.assertEqual(list(result.columns), ['a', 'b']) + def test_move_column_left_same_column(self): + """Tests that move_column_left safely handles column_to_move equal to target_column.""" + df = pd.DataFrame({'a': [1], 'b': [2], 'c': [3]}) + result = preprocess.move_column_left(df, 'b', 'b') + self.assertEqual(list(result.columns), ['a', 'b', 'c']) + def test_preprocess_data(self): """Tests data preprocessing and splitting into age-only and general survey CSVs.""" with tempfile.TemporaryDirectory() as tmp_dir: @@ -98,12 +104,38 @@ def test_preprocess_data(self): for age_col in preprocess.AGE_COLUMNS: self.assertNotIn(age_col, cols_data) + @mock.patch('preprocess.logging.fatal') + def test_preprocess_data_file_not_found(self, mock_fatal): + """Tests that preprocess_data exits with code 1 if input file is missing.""" + with tempfile.TemporaryDirectory() as tmp_dir: + missing_input = os.path.join(tmp_dir, 'nonexistent.csv') + with mock.patch.object(preprocess, 'INPUT_DIR', tmp_dir), \ + mock.patch.object(preprocess, 'INPUT_FILE', missing_input): + with self.assertRaises(SystemExit) as cm: + preprocess.preprocess_data() + self.assertEqual(cm.exception.code, 1) + self.assertTrue(mock_fatal.called) + + @mock.patch('preprocess.logging.fatal') + def test_preprocess_data_missing_columns(self, mock_fatal): + """Tests that preprocess_data exits with code 1 if input CSV lacks required columns.""" + with tempfile.TemporaryDirectory() as tmp_dir: + bad_input = os.path.join(tmp_dir, 'bad.csv') + pd.DataFrame({'incomplete': [1, 2]}).to_csv(bad_input, index=False) + with mock.patch.object(preprocess, 'INPUT_DIR', tmp_dir), \ + mock.patch.object(preprocess, 'INPUT_FILE', bad_input): + with self.assertRaises(SystemExit) as cm: + preprocess.preprocess_data() + self.assertEqual(cm.exception.code, 1) + self.assertTrue(mock_fatal.called) + @mock.patch('preprocess.preprocess_data') @mock.patch('preprocess.download_file') def test_main_download_success(self, mock_download, mock_preprocess): """Tests that main downloads file and executes preprocess_data on success.""" mock_download.return_value = True - with mock.patch('os.path.exists', return_value=True): + with mock.patch('os.path.exists', return_value=True), \ + mock.patch('os.path.getsize', return_value=1024): preprocess.main([]) mock_download.assert_called_once_with( url=preprocess.Commerce_NTIA_URL, @@ -121,21 +153,56 @@ def test_main_download_success(self, mock_download, mock_preprocess): @mock.patch('preprocess.logging.fatal') def test_main_download_failure(self, mock_fatal, mock_download, mock_preprocess): - """Tests that main logs fatal error and halts when download returns False.""" + """Tests that main logs fatal error and exits with code 1 when download returns False.""" mock_download.return_value = False - preprocess.main([]) + with self.assertRaises(SystemExit) as cm: + preprocess.main([]) + self.assertEqual(cm.exception.code, 1) mock_fatal.assert_called_once_with( - "Failed to download Commerce_NTIA file.") + "Failed to download Commerce_NTIA file or file is empty.") mock_preprocess.assert_not_called() + @mock.patch('preprocess.preprocess_data') + @mock.patch('preprocess.download_file') + @mock.patch('preprocess.logging.fatal') + def test_main_download_success_file_missing(self, mock_fatal, mock_download, + mock_preprocess): + """Tests that main exits with code 1 if download reports success but file is missing.""" + mock_download.return_value = True + with mock.patch('os.path.exists', return_value=False): + with self.assertRaises(SystemExit) as cm: + preprocess.main([]) + self.assertEqual(cm.exception.code, 1) + mock_fatal.assert_called_once_with( + "Failed to download Commerce_NTIA file or file is empty.") + mock_preprocess.assert_not_called() + + @mock.patch('preprocess.preprocess_data') + @mock.patch('preprocess.download_file') + @mock.patch('preprocess.logging.fatal') + def test_main_download_success_empty_file(self, mock_fatal, mock_download, + mock_preprocess): + """Tests that main exits with code 1 if downloaded file is 0 bytes.""" + mock_download.return_value = True + with mock.patch('os.path.exists', return_value=True), \ + mock.patch('os.path.getsize', return_value=0): + with self.assertRaises(SystemExit) as cm: + preprocess.main([]) + self.assertEqual(cm.exception.code, 1) + mock_fatal.assert_called_once_with( + "Failed to download Commerce_NTIA file or file is empty.") + mock_preprocess.assert_not_called() + @mock.patch('preprocess.preprocess_data') @mock.patch('preprocess.download_file') @mock.patch('preprocess.logging.fatal') def test_main_download_exception(self, mock_fatal, mock_download, mock_preprocess): - """Tests that main logs fatal error when download raises an exception.""" + """Tests that main logs fatal error and exits with code 1 when download raises an exception.""" mock_download.side_effect = Exception("Connection timeout") - preprocess.main([]) + with self.assertRaises(SystemExit) as cm: + preprocess.main([]) + self.assertEqual(cm.exception.code, 1) self.assertTrue(mock_fatal.called) self.assertIn("Connection timeout", str(mock_fatal.call_args)) mock_preprocess.assert_not_called() diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py b/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py index 6ca9a532be..6ec9f4daa7 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py @@ -30,8 +30,7 @@ COMMON_COLUMNS = ["dataset", "variable", "description", "universe"] AGE_COLUMNS = [ - "age314Count", "age1524Count", "age2544Count", "age4564Count", - "age65pCount" + "age314Count", "age1524Count", "age2544Count", "age4564Count", "age65pCount" ] INPUT_FILE = os.path.join(INPUT_DIR, "ntia-analyze-table.csv") INPUT_FILE_1 = os.path.join(INPUT_DIR, "ntia-data-age-only.csv") @@ -40,13 +39,14 @@ HEADERS = { 'User-Agent': ('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'), - 'Accept': - 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', } def move_column_left(df, column_to_move, target_column): """Moves the universe column to the left of variable column.""" + if column_to_move == target_column: + return df cols = df.columns.tolist() if column_to_move in cols and target_column in cols: cols.remove(column_to_move) @@ -93,6 +93,7 @@ def preprocess_data(): def main(argv): + del argv try: success = download_file( url=Commerce_NTIA_URL, @@ -103,12 +104,14 @@ def main(argv): delay=5, backoff=2, ) - if not success or not os.path.exists(INPUT_FILE): - logging.fatal("Failed to download Commerce_NTIA file.") - return + if not success or not os.path.exists(INPUT_FILE) or os.path.getsize( + INPUT_FILE) == 0: + logging.fatal( + "Failed to download Commerce_NTIA file or file is empty.") + sys.exit(1) except Exception as e: logging.fatal(f"Failed to download Commerce_NTIA file: {e}") - return + sys.exit(1) preprocess_data() From e780f0d09fdc21ed785813651e44cc898ce17d1b Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Mon, 14 Sep 2026 08:08:08 +0000 Subject: [PATCH 4/7] Made changes suggested by CRAs analysis --- .../commerce_ntia/commerce_ntia_test.py | 7 ++++++- .../ntia_internet_use_survey/commerce_ntia/preprocess.py | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py b/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py index 0d4bbba7ad..28a39edbb3 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py @@ -62,10 +62,13 @@ def test_preprocess_data(self): 'age2544Count': [12, 22, 32], 'age4564Count': [13, 23, 33], 'age65pCount': [14, 24, 34], + 'agencyAccess': [5, 10, 15], 'totalCount': [100, 200, 300], 'otherMetric': [1.5, 2.5, 3.5] } - pd.DataFrame(raw_data).to_csv(input_file, index=False) + pd.DataFrame(raw_data).to_csv(input_file, + index=False, + encoding='utf-8-sig') with mock.patch.object(preprocess, 'INPUT_DIR', tmp_dir), \ mock.patch.object(preprocess, 'INPUT_FILE', input_file), \ @@ -82,6 +85,7 @@ def test_preprocess_data(self): cols_age.index('universe') + 1, cols_age.index('variable')) for age_col in preprocess.AGE_COLUMNS: self.assertIn(age_col, cols_age) + self.assertNotIn('agencyAccess', cols_age) self.assertNotIn('totalCount', cols_age) self.assertNotIn('otherMetric', cols_age) self.assertIn('universeAgeResol', cols_age) @@ -94,6 +98,7 @@ def test_preprocess_data(self): cols_data = list(df_data.columns) self.assertEqual( cols_data.index('universe') + 1, cols_data.index('variable')) + self.assertIn('agencyAccess', cols_data) self.assertIn('totalCount', cols_data) self.assertIn('otherMetric', cols_data) self.assertIn('universeAgeResol', cols_data) diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py b/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py index 6ec9f4daa7..423c611f8c 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py @@ -59,7 +59,7 @@ def move_column_left(df, column_to_move, target_column): def preprocess_data(): try: os.makedirs(INPUT_DIR, exist_ok=True) - org_df = pd.read_csv(INPUT_FILE) + org_df = pd.read_csv(INPUT_FILE, encoding='utf-8-sig') # 1. Process Age-only data df1 = org_df[COMMON_COLUMNS + AGE_COLUMNS].copy() @@ -74,7 +74,7 @@ def preprocess_data(): # 2. Process General survey data df2_cols_to_keep = [ - col for col in org_df.columns if not col.startswith('age') + col for col in org_df.columns if col not in set(AGE_COLUMNS) ] df2 = org_df[df2_cols_to_keep].copy() df2['universeAgeResol'] = df2['universe'].apply( From 5647a46f061d372ff517816f20ad44af423f8fd6 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Wed, 16 Sep 2026 21:33:18 +0000 Subject: [PATCH 5/7] Made changes as perreview comments --- .../commerce_ntia/commerce_ntia_test.py | 12 ++++++++++ .../commerce_ntia/manifest.json | 4 ++-- .../commerce_ntia/preprocess.py | 24 +++++++------------ .../commerce_ntia/validation_config.json | 17 +++++++++---- 4 files changed, 35 insertions(+), 22 deletions(-) diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py b/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py index 28a39edbb3..7bd7fc9ae8 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py @@ -58,10 +58,12 @@ def test_preprocess_data(self): 'description': ['Desc 1', 'Desc 2', 'Desc 3'], 'universe': ['isPerson', 'isAdult', 'isHousehold'], 'age314Count': [10, 20, 30], + 'age314Prop': [0.1, 0.2, 0.3], 'age1524Count': [11, 21, 31], 'age2544Count': [12, 22, 32], 'age4564Count': [13, 23, 33], 'age65pCount': [14, 24, 34], + 'age65pSE': [0.01, 0.02, 0.03], 'agencyAccess': [5, 10, 15], 'totalCount': [100, 200, 300], 'otherMetric': [1.5, 2.5, 3.5] @@ -85,6 +87,8 @@ def test_preprocess_data(self): cols_age.index('universe') + 1, cols_age.index('variable')) for age_col in preprocess.AGE_COLUMNS: self.assertIn(age_col, cols_age) + self.assertNotIn('age314Prop', cols_age) + self.assertNotIn('age65pSE', cols_age) self.assertNotIn('agencyAccess', cols_age) self.assertNotIn('totalCount', cols_age) self.assertNotIn('otherMetric', cols_age) @@ -108,10 +112,13 @@ def test_preprocess_data(self): self.assertTrue(pd.isna(df_data.loc[2, 'universeAgeResol'])) for age_col in preprocess.AGE_COLUMNS: self.assertNotIn(age_col, cols_data) + self.assertNotIn('age314Prop', cols_data) + self.assertNotIn('age65pSE', cols_data) @mock.patch('preprocess.logging.fatal') def test_preprocess_data_file_not_found(self, mock_fatal): """Tests that preprocess_data exits with code 1 if input file is missing.""" + mock_fatal.side_effect = SystemExit(1) with tempfile.TemporaryDirectory() as tmp_dir: missing_input = os.path.join(tmp_dir, 'nonexistent.csv') with mock.patch.object(preprocess, 'INPUT_DIR', tmp_dir), \ @@ -124,6 +131,7 @@ def test_preprocess_data_file_not_found(self, mock_fatal): @mock.patch('preprocess.logging.fatal') def test_preprocess_data_missing_columns(self, mock_fatal): """Tests that preprocess_data exits with code 1 if input CSV lacks required columns.""" + mock_fatal.side_effect = SystemExit(1) with tempfile.TemporaryDirectory() as tmp_dir: bad_input = os.path.join(tmp_dir, 'bad.csv') pd.DataFrame({'incomplete': [1, 2]}).to_csv(bad_input, index=False) @@ -159,6 +167,7 @@ def test_main_download_success(self, mock_download, mock_preprocess): def test_main_download_failure(self, mock_fatal, mock_download, mock_preprocess): """Tests that main logs fatal error and exits with code 1 when download returns False.""" + mock_fatal.side_effect = SystemExit(1) mock_download.return_value = False with self.assertRaises(SystemExit) as cm: preprocess.main([]) @@ -173,6 +182,7 @@ def test_main_download_failure(self, mock_fatal, mock_download, def test_main_download_success_file_missing(self, mock_fatal, mock_download, mock_preprocess): """Tests that main exits with code 1 if download reports success but file is missing.""" + mock_fatal.side_effect = SystemExit(1) mock_download.return_value = True with mock.patch('os.path.exists', return_value=False): with self.assertRaises(SystemExit) as cm: @@ -188,6 +198,7 @@ def test_main_download_success_file_missing(self, mock_fatal, mock_download, def test_main_download_success_empty_file(self, mock_fatal, mock_download, mock_preprocess): """Tests that main exits with code 1 if downloaded file is 0 bytes.""" + mock_fatal.side_effect = SystemExit(1) mock_download.return_value = True with mock.patch('os.path.exists', return_value=True), \ mock.patch('os.path.getsize', return_value=0): @@ -204,6 +215,7 @@ def test_main_download_success_empty_file(self, mock_fatal, mock_download, def test_main_download_exception(self, mock_fatal, mock_download, mock_preprocess): """Tests that main logs fatal error and exits with code 1 when download raises an exception.""" + mock_fatal.side_effect = SystemExit(1) mock_download.side_effect = Exception("Connection timeout") with self.assertRaises(SystemExit) as cm: preprocess.main([]) diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json b/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json index ca3902d612..a17fdaa771 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json @@ -20,12 +20,12 @@ { "template_mcf": "output_files/ntia_output.tmcf", "cleaned_csv": "output_files/ntia_output.csv", - "node_mcf": "output_files/*.mcf" + "node_mcf": "output_files/ntia_output*.mcf" }, { "template_mcf": "output_files/ntia_age_output.tmcf", "cleaned_csv": "output_files/ntia_age_output.csv", - "node_mcf": "output_files/*.mcf" + "node_mcf": "output_files/ntia_age_output*.mcf" } ], "cron_schedule": "0 06 * * 5", diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py b/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py index 423c611f8c..8034b12073 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py @@ -32,6 +32,7 @@ AGE_COLUMNS = [ "age314Count", "age1524Count", "age2544Count", "age4564Count", "age65pCount" ] +_AGE_RESOL_MAP = {'isPerson': 'CivilPerson', 'isAdult': 'Adult'} INPUT_FILE = os.path.join(INPUT_DIR, "ntia-analyze-table.csv") INPUT_FILE_1 = os.path.join(INPUT_DIR, "ntia-data-age-only.csv") INPUT_FILE_2 = os.path.join(INPUT_DIR, "ntia-data.csv") @@ -63,33 +64,26 @@ def preprocess_data(): # 1. Process Age-only data df1 = org_df[COMMON_COLUMNS + AGE_COLUMNS].copy() - df1['universeAgeResol'] = df1['universe'].apply( - lambda x: 'CivilPerson' - if x == 'isPerson' else ('Adult' if x == 'isAdult' else None)) - df1['variableAgeResol'] = df1['variable'].apply( - lambda x: 'CivilPerson' - if x == 'isPerson' else ('Adult' if x == 'isAdult' else None)) + df1['universeAgeResol'] = df1['universe'].map(_AGE_RESOL_MAP) + df1['variableAgeResol'] = df1['variable'].map(_AGE_RESOL_MAP) df1_moved = move_column_left(df1, 'universe', 'variable') df1_moved.to_csv(INPUT_FILE_1, index=False) # 2. Process General survey data df2_cols_to_keep = [ - col for col in org_df.columns if col not in set(AGE_COLUMNS) + col for col in org_df.columns + if not col.startswith( + ('age314', 'age1524', 'age2544', 'age4564', 'age65p')) ] df2 = org_df[df2_cols_to_keep].copy() - df2['universeAgeResol'] = df2['universe'].apply( - lambda x: 'CivilPerson' - if x == 'isPerson' else ('Adult' if x == 'isAdult' else None)) - df2['variableAgeResol'] = df2['variable'].apply( - lambda x: 'CivilPerson' - if x == 'isPerson' else ('Adult' if x == 'isAdult' else None)) + df2['universeAgeResol'] = df2['universe'].map(_AGE_RESOL_MAP) + df2['variableAgeResol'] = df2['variable'].map(_AGE_RESOL_MAP) df2_moved = move_column_left(df2, 'universe', 'variable') df2_moved.to_csv(INPUT_FILE_2, index=False) except Exception as e: logging.fatal( f"An error occurred while preprocessing the input data: {e}") - sys.exit(1) def main(argv): @@ -108,10 +102,8 @@ def main(argv): INPUT_FILE) == 0: logging.fatal( "Failed to download Commerce_NTIA file or file is empty.") - sys.exit(1) except Exception as e: logging.fatal(f"Failed to download Commerce_NTIA file: {e}") - sys.exit(1) preprocess_data() diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/validation_config.json b/statvar_imports/ntia_internet_use_survey/commerce_ntia/validation_config.json index c94065b0c2..de76934934 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/validation_config.json +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/validation_config.json @@ -10,12 +10,21 @@ } }, { - "rule_id": "check_max_date_freshness", - "description": "Checks that the latest observation date is at least 2023.", + "rule_id": "check_active_survey_wave_count", + "description": "Checks that all active StatVars (1935 in input0, 227 in input1) have MaxDate >= 2023-11.", "validator": "SQL_VALIDATOR", "params": { - "query": "SELECT MAX(MaxDate) AS max_date FROM stats", - "condition": "max_date >= '2023'" + "query": "SELECT COUNT(*) AS total_svs, SUM(CASE WHEN MaxDate >= '2023-11' THEN 1 ELSE 0 END) AS active_svs FROM stats", + "condition": "active_svs >= CASE WHEN total_svs > 500 THEN 1935 ELSE 227 END" + } + }, + { + "rule_id": "check_statvar_min_max_date", + "description": "Checks that no StatVar regresses below the oldest historical wave (2013-07).", + "validator": "SQL_VALIDATOR", + "params": { + "query": "SELECT StatVar, MaxDate FROM stats", + "condition": "MaxDate >= '2013-07'" } } ] From a49cff6347d29547cda1baa77ed02554df0dbb7e Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Thu, 17 Sep 2026 06:19:34 +0000 Subject: [PATCH 6/7] Modified validation_config.json --- .../commerce_ntia/validation_config.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/validation_config.json b/statvar_imports/ntia_internet_use_survey/commerce_ntia/validation_config.json index de76934934..4fb27de14d 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/validation_config.json +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/validation_config.json @@ -14,8 +14,8 @@ "description": "Checks that all active StatVars (1935 in input0, 227 in input1) have MaxDate >= 2023-11.", "validator": "SQL_VALIDATOR", "params": { - "query": "SELECT COUNT(*) AS total_svs, SUM(CASE WHEN MaxDate >= '2023-11' THEN 1 ELSE 0 END) AS active_svs FROM stats", - "condition": "active_svs >= CASE WHEN total_svs > 500 THEN 1935 ELSE 227 END" + "query": "SELECT COUNT(*) AS total_svs, COALESCE(SUM(CASE WHEN MaxDate >= '2023-11' THEN 1 ELSE 0 END), 0) AS active_svs FROM stats", + "condition": "(total_svs >= 1935 AND active_svs >= 1935) OR (total_svs BETWEEN 227 AND 300 AND active_svs >= 227)" } }, { @@ -24,7 +24,7 @@ "validator": "SQL_VALIDATOR", "params": { "query": "SELECT StatVar, MaxDate FROM stats", - "condition": "MaxDate >= '2013-07'" + "condition": "MaxDate IS NOT NULL AND MaxDate >= '2013-07'" } } ] From 6de3fac3e12ce24ba326cef8ef242a3964e8a602 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Thu, 17 Sep 2026 09:39:14 +0000 Subject: [PATCH 7/7] Modified validation_config.json rules --- .../commerce_ntia/validation_config.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/validation_config.json b/statvar_imports/ntia_internet_use_survey/commerce_ntia/validation_config.json index 4fb27de14d..c67ef0794c 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/validation_config.json +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/validation_config.json @@ -14,8 +14,8 @@ "description": "Checks that all active StatVars (1935 in input0, 227 in input1) have MaxDate >= 2023-11.", "validator": "SQL_VALIDATOR", "params": { - "query": "SELECT COUNT(*) AS total_svs, COALESCE(SUM(CASE WHEN MaxDate >= '2023-11' THEN 1 ELSE 0 END), 0) AS active_svs FROM stats", - "condition": "(total_svs >= 1935 AND active_svs >= 1935) OR (total_svs BETWEEN 227 AND 300 AND active_svs >= 227)" + "query": "SELECT COUNT(*) AS total_svs, COALESCE(SUM(CASE WHEN MaxDate >= '2023-11' THEN 1 ELSE 0 END), 0) AS active_svs, COALESCE(SUM(CASE WHEN regexp_matches(StatVar, '3To14Years|15To24Years|25To44Years|45To64Years|65OrMoreYears') THEN 1 ELSE 0 END), 0) AS age_svs FROM stats", + "condition": "(age_svs = 0 AND total_svs >= 1935 AND active_svs >= 1935) OR (age_svs = total_svs AND total_svs BETWEEN 227 AND 300 AND active_svs >= 227)" } }, {