diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md b/statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md index eb6e1e8ea1..0696662625 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md @@ -5,8 +5,8 @@ - NTIA programs and policymaking focus largely on expanding broadband Internet access and adoption in America, expanding the use of spectrum by all users. - how to download data: - To download and process the data, you'll need to run the provided preprocess script, `preprocess.py`. This script will automatically create an "input_files" folder where you should place the file to be processed. - By using this script, we are creating two more columns in the input files such as 'universeAgeResol', 'variableAgeResol'. This columns are created based on the universe and variable columns in the existing data. + To download and process the data, you'll need to run the provided preprocess script, `preprocess.py`. This script will automatically create an "input_files" folder and download the file to be processed. + This script organizes the data and splits it into general survey data and age-breakdown data. - type of place: Demographics. @@ -16,9 +16,10 @@ ``` python3 stat_var_processor.py ---input_data='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/input_files/' ---pv_map='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/' --config_file='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/' --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf ---output_path='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA//' +--input_data='../../statvar_imports/ntia_internet_use_survey/commerce_ntia/input_files/' +--pv_map='../../statvar_imports/ntia_internet_use_survey/commerce_ntia/' --config_file='../../statvar_imports/ntia_internet_use_survey/commerce_ntia/' --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf +--output_path='../../statvar_imports/ntia_internet_use_survey/commerce_ntia//' +--output_counters='../../statvar_imports/ntia_internet_use_survey/commerce_ntia//' ``` #### Download the data: @@ -36,17 +37,19 @@ Execute the script inside the folder `/data/tools/statvar_importer/` ``` python3 stat_var_processor.py ---input_data=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/input_files/ntia-data.csv ---pv_map=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_pvmap.csv ---config_file=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf ---output_path=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/output_files/ntia_output +--input_data=../../statvar_imports/ntia_internet_use_survey/commerce_ntia/input_files/ntia-data.csv +--pv_map=../../statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_pvmap.csv +--config_file=../../statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf +--output_path=../../statvar_imports/ntia_internet_use_survey/commerce_ntia/output_files/ntia_output +--output_counters=../../statvar_imports/ntia_internet_use_survey/commerce_ntia/counters/ntia_output_counters.csv ``` ``` python3 stat_var_processor.py ---input_data=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/input_files/ntia-data-age-only.csv ---pv_map=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_age_pvmap.csv ---config_file=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf ---output_path=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/output_files/ntia_age_output +--input_data=../../statvar_imports/ntia_internet_use_survey/commerce_ntia/input_files/ntia-data-age-only.csv +--pv_map=../../statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_age_pvmap.csv +--config_file=../../statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf +--output_path=../../statvar_imports/ntia_internet_use_survey/commerce_ntia/output_files/ntia_age_output +--output_counters=../../statvar_imports/ntia_internet_use_survey/commerce_ntia/counters/ntia_age_output_counters.csv ``` diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py b/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py new file mode 100644 index 0000000000..7bd7fc9ae8 --- /dev/null +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py @@ -0,0 +1,229 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Hermetic unit tests for commerce_ntia preprocess module.""" + +import os +import sys +import tempfile +import unittest +from unittest import mock +import pandas as pd + +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, _SCRIPT_DIR) +import preprocess + + +class PreprocessTest(unittest.TestCase): + + def test_move_column_left_success(self): + """Tests that move_column_left places column immediately left of target.""" + df = pd.DataFrame({'a': [1], 'b': [2], 'c': [3], 'd': [4]}) + result = preprocess.move_column_left(df, 'd', 'b') + self.assertEqual(list(result.columns), ['a', 'd', 'b', 'c']) + + def test_move_column_left_missing_cols(self): + """Tests that move_column_left returns original df if columns are not present.""" + df = pd.DataFrame({'a': [1], 'b': [2]}) + result = preprocess.move_column_left(df, 'missing', 'b') + self.assertEqual(list(result.columns), ['a', 'b']) + + def test_move_column_left_same_column(self): + """Tests that move_column_left safely handles column_to_move equal to target_column.""" + df = pd.DataFrame({'a': [1], 'b': [2], 'c': [3]}) + result = preprocess.move_column_left(df, 'b', 'b') + self.assertEqual(list(result.columns), ['a', 'b', 'c']) + + def test_preprocess_data(self): + """Tests data preprocessing and splitting into age-only and general survey CSVs.""" + with tempfile.TemporaryDirectory() as tmp_dir: + input_file = os.path.join(tmp_dir, 'ntia-analyze-table.csv') + output_age = os.path.join(tmp_dir, 'ntia-data-age-only.csv') + output_data = os.path.join(tmp_dir, 'ntia-data.csv') + + raw_data = { + 'dataset': ['Nov 2023', 'Nov 2023', 'Nov 2023'], + 'variable': ['Streaming', 'Email', 'Broadband'], + 'description': ['Desc 1', 'Desc 2', 'Desc 3'], + 'universe': ['isPerson', 'isAdult', 'isHousehold'], + 'age314Count': [10, 20, 30], + 'age314Prop': [0.1, 0.2, 0.3], + 'age1524Count': [11, 21, 31], + 'age2544Count': [12, 22, 32], + 'age4564Count': [13, 23, 33], + 'age65pCount': [14, 24, 34], + 'age65pSE': [0.01, 0.02, 0.03], + 'agencyAccess': [5, 10, 15], + 'totalCount': [100, 200, 300], + 'otherMetric': [1.5, 2.5, 3.5] + } + pd.DataFrame(raw_data).to_csv(input_file, + index=False, + encoding='utf-8-sig') + + with mock.patch.object(preprocess, 'INPUT_DIR', tmp_dir), \ + mock.patch.object(preprocess, 'INPUT_FILE', input_file), \ + mock.patch.object(preprocess, 'INPUT_FILE_1', output_age), \ + mock.patch.object(preprocess, 'INPUT_FILE_2', output_data): + preprocess.preprocess_data() + + self.assertTrue(os.path.exists(output_age)) + self.assertTrue(os.path.exists(output_data)) + + df_age = pd.read_csv(output_age) + cols_age = list(df_age.columns) + self.assertEqual( + cols_age.index('universe') + 1, cols_age.index('variable')) + for age_col in preprocess.AGE_COLUMNS: + self.assertIn(age_col, cols_age) + self.assertNotIn('age314Prop', cols_age) + self.assertNotIn('age65pSE', cols_age) + self.assertNotIn('agencyAccess', cols_age) + self.assertNotIn('totalCount', cols_age) + self.assertNotIn('otherMetric', cols_age) + self.assertIn('universeAgeResol', cols_age) + self.assertIn('variableAgeResol', cols_age) + self.assertEqual(df_age.loc[0, 'universeAgeResol'], 'CivilPerson') + self.assertEqual(df_age.loc[1, 'universeAgeResol'], 'Adult') + self.assertTrue(pd.isna(df_age.loc[2, 'universeAgeResol'])) + + df_data = pd.read_csv(output_data) + cols_data = list(df_data.columns) + self.assertEqual( + cols_data.index('universe') + 1, cols_data.index('variable')) + self.assertIn('agencyAccess', cols_data) + self.assertIn('totalCount', cols_data) + self.assertIn('otherMetric', cols_data) + self.assertIn('universeAgeResol', cols_data) + self.assertIn('variableAgeResol', cols_data) + self.assertEqual(df_data.loc[0, 'universeAgeResol'], 'CivilPerson') + self.assertEqual(df_data.loc[1, 'universeAgeResol'], 'Adult') + self.assertTrue(pd.isna(df_data.loc[2, 'universeAgeResol'])) + for age_col in preprocess.AGE_COLUMNS: + self.assertNotIn(age_col, cols_data) + self.assertNotIn('age314Prop', cols_data) + self.assertNotIn('age65pSE', cols_data) + + @mock.patch('preprocess.logging.fatal') + def test_preprocess_data_file_not_found(self, mock_fatal): + """Tests that preprocess_data exits with code 1 if input file is missing.""" + mock_fatal.side_effect = SystemExit(1) + with tempfile.TemporaryDirectory() as tmp_dir: + missing_input = os.path.join(tmp_dir, 'nonexistent.csv') + with mock.patch.object(preprocess, 'INPUT_DIR', tmp_dir), \ + mock.patch.object(preprocess, 'INPUT_FILE', missing_input): + with self.assertRaises(SystemExit) as cm: + preprocess.preprocess_data() + self.assertEqual(cm.exception.code, 1) + self.assertTrue(mock_fatal.called) + + @mock.patch('preprocess.logging.fatal') + def test_preprocess_data_missing_columns(self, mock_fatal): + """Tests that preprocess_data exits with code 1 if input CSV lacks required columns.""" + mock_fatal.side_effect = SystemExit(1) + with tempfile.TemporaryDirectory() as tmp_dir: + bad_input = os.path.join(tmp_dir, 'bad.csv') + pd.DataFrame({'incomplete': [1, 2]}).to_csv(bad_input, index=False) + with mock.patch.object(preprocess, 'INPUT_DIR', tmp_dir), \ + mock.patch.object(preprocess, 'INPUT_FILE', bad_input): + with self.assertRaises(SystemExit) as cm: + preprocess.preprocess_data() + self.assertEqual(cm.exception.code, 1) + self.assertTrue(mock_fatal.called) + + @mock.patch('preprocess.preprocess_data') + @mock.patch('preprocess.download_file') + def test_main_download_success(self, mock_download, mock_preprocess): + """Tests that main downloads file and executes preprocess_data on success.""" + mock_download.return_value = True + with mock.patch('os.path.exists', return_value=True), \ + mock.patch('os.path.getsize', return_value=1024): + preprocess.main([]) + mock_download.assert_called_once_with( + url=preprocess.Commerce_NTIA_URL, + output_folder=preprocess.INPUT_DIR, + unzip=False, + headers=preprocess.HEADERS, + tries=3, + delay=5, + backoff=2, + ) + mock_preprocess.assert_called_once() + + @mock.patch('preprocess.preprocess_data') + @mock.patch('preprocess.download_file') + @mock.patch('preprocess.logging.fatal') + def test_main_download_failure(self, mock_fatal, mock_download, + mock_preprocess): + """Tests that main logs fatal error and exits with code 1 when download returns False.""" + mock_fatal.side_effect = SystemExit(1) + mock_download.return_value = False + with self.assertRaises(SystemExit) as cm: + preprocess.main([]) + self.assertEqual(cm.exception.code, 1) + mock_fatal.assert_called_once_with( + "Failed to download Commerce_NTIA file or file is empty.") + mock_preprocess.assert_not_called() + + @mock.patch('preprocess.preprocess_data') + @mock.patch('preprocess.download_file') + @mock.patch('preprocess.logging.fatal') + def test_main_download_success_file_missing(self, mock_fatal, mock_download, + mock_preprocess): + """Tests that main exits with code 1 if download reports success but file is missing.""" + mock_fatal.side_effect = SystemExit(1) + mock_download.return_value = True + with mock.patch('os.path.exists', return_value=False): + with self.assertRaises(SystemExit) as cm: + preprocess.main([]) + self.assertEqual(cm.exception.code, 1) + mock_fatal.assert_called_once_with( + "Failed to download Commerce_NTIA file or file is empty.") + mock_preprocess.assert_not_called() + + @mock.patch('preprocess.preprocess_data') + @mock.patch('preprocess.download_file') + @mock.patch('preprocess.logging.fatal') + def test_main_download_success_empty_file(self, mock_fatal, mock_download, + mock_preprocess): + """Tests that main exits with code 1 if downloaded file is 0 bytes.""" + mock_fatal.side_effect = SystemExit(1) + mock_download.return_value = True + with mock.patch('os.path.exists', return_value=True), \ + mock.patch('os.path.getsize', return_value=0): + with self.assertRaises(SystemExit) as cm: + preprocess.main([]) + self.assertEqual(cm.exception.code, 1) + mock_fatal.assert_called_once_with( + "Failed to download Commerce_NTIA file or file is empty.") + mock_preprocess.assert_not_called() + + @mock.patch('preprocess.preprocess_data') + @mock.patch('preprocess.download_file') + @mock.patch('preprocess.logging.fatal') + def test_main_download_exception(self, mock_fatal, mock_download, + mock_preprocess): + """Tests that main logs fatal error and exits with code 1 when download raises an exception.""" + mock_fatal.side_effect = SystemExit(1) + mock_download.side_effect = Exception("Connection timeout") + with self.assertRaises(SystemExit) as cm: + preprocess.main([]) + self.assertEqual(cm.exception.code, 1) + self.assertTrue(mock_fatal.called) + self.assertIn("Connection timeout", str(mock_fatal.call_args)) + mock_preprocess.assert_not_called() + + +if __name__ == '__main__': + unittest.main() diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json b/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json index 2906cb50c3..a17fdaa771 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json @@ -9,23 +9,27 @@ "provenance_description": "NTIA programs and policymaking focus largely on expanding broadband Internet access and adoption in America, expanding the use of spectrum by all users.", "scripts": [ "preprocess.py", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ntia-data.csv --pv_map=ntia_pvmap.csv --config_file=ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ntia_output", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ntia-data-age-only.csv --pv_map=ntia_age_pvmap.csv --config_file=ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ntia_age_output" + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ntia-data.csv --pv_map=ntia_pvmap.csv --config_file=ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ntia_output --output_counters=counters/ntia_output_counters.csv", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ntia-data-age-only.csv --pv_map=ntia_age_pvmap.csv --config_file=ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ntia_age_output --output_counters=counters/ntia_age_output_counters.csv" ], "source_files": [ - "input_files/ntia-analyze-table.csv" + "input_files/ntia-analyze-table.csv", + "counters/*.csv" ], "import_inputs": [ { "template_mcf": "output_files/ntia_output.tmcf", - "cleaned_csv": "output_files/ntia_output.csv" + "cleaned_csv": "output_files/ntia_output.csv", + "node_mcf": "output_files/ntia_output*.mcf" }, { "template_mcf": "output_files/ntia_age_output.tmcf", - "cleaned_csv": "output_files/ntia_age_output.csv" + "cleaned_csv": "output_files/ntia_age_output.csv", + "node_mcf": "output_files/ntia_age_output*.mcf" } ], - "cron_schedule": "0 06 * * 5" + "cron_schedule": "0 06 * * 5", + "validation_config_file": "validation_config.json" } ] } \ No newline at end of file diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py b/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py index 9bcf786a14..8034b12073 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os, sys +import os +import sys import pandas as pd from absl import app, logging -from pathlib import Path import config script_dir = os.path.dirname(os.path.abspath(__file__)) @@ -27,18 +27,27 @@ Commerce_NTIA_URL = config.Commerce_NTIA_URL INPUT_DIR = os.path.join(script_dir, "input_files") -Path(INPUT_DIR).mkdir(parents=True, exist_ok=True) - COMMON_COLUMNS = ["dataset", "variable", "description", "universe"] -AGE_COLUMNS = ["age314Count", "age1524Count", "age2544Count", "age4564Count", "age65pCount"] +AGE_COLUMNS = [ + "age314Count", "age1524Count", "age2544Count", "age4564Count", "age65pCount" +] +_AGE_RESOL_MAP = {'isPerson': 'CivilPerson', 'isAdult': 'Adult'} INPUT_FILE = os.path.join(INPUT_DIR, "ntia-analyze-table.csv") INPUT_FILE_1 = os.path.join(INPUT_DIR, "ntia-data-age-only.csv") INPUT_FILE_2 = os.path.join(INPUT_DIR, "ntia-data.csv") +HEADERS = { + 'User-Agent': ('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'), + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', +} + def move_column_left(df, column_to_move, target_column): """Moves the universe column to the left of variable column.""" + if column_to_move == target_column: + return df cols = df.columns.tolist() if column_to_move in cols and target_column in cols: cols.remove(column_to_move) @@ -50,45 +59,54 @@ def move_column_left(df, column_to_move, target_column): def preprocess_data(): try: - org_df = pd.read_csv(INPUT_FILE) - + os.makedirs(INPUT_DIR, exist_ok=True) + org_df = pd.read_csv(INPUT_FILE, encoding='utf-8-sig') + + # 1. Process Age-only data df1 = org_df[COMMON_COLUMNS + AGE_COLUMNS].copy() - df1['universeAgeResol'] = df1['universe'].apply( - lambda x: 'CivilPerson' if x == 'isPerson' else ('Adult' if x == 'isAdult' else None) - ) - df1['variableAgeResol'] = df1['variable'].apply( - lambda x: 'CivilPerson' if x == 'isPerson' else ('Adult' if x == 'isAdult' else None) - ) + df1['universeAgeResol'] = df1['universe'].map(_AGE_RESOL_MAP) + df1['variableAgeResol'] = df1['variable'].map(_AGE_RESOL_MAP) df1_moved = move_column_left(df1, 'universe', 'variable') df1_moved.to_csv(INPUT_FILE_1, index=False) - df2_cols_to_keep = [col for col in org_df.columns if not col.startswith('age')] + # 2. Process General survey data + df2_cols_to_keep = [ + col for col in org_df.columns + if not col.startswith( + ('age314', 'age1524', 'age2544', 'age4564', 'age65p')) + ] df2 = org_df[df2_cols_to_keep].copy() - df2['universeAgeResol'] = df2['universe'].apply( - lambda x: 'CivilPerson' if x == 'isPerson' else ('Adult' if x == 'isAdult' else None) - ) - df2['variableAgeResol'] = df2['variable'].apply( - lambda x: 'CivilPerson' if x == 'isPerson' else ('Adult' if x == 'isAdult' else None) - ) + df2['universeAgeResol'] = df2['universe'].map(_AGE_RESOL_MAP) + df2['variableAgeResol'] = df2['variable'].map(_AGE_RESOL_MAP) df2_moved = move_column_left(df2, 'universe', 'variable') df2_moved.to_csv(INPUT_FILE_2, index=False) except Exception as e: - logging.fatal(f"An error occurred while preprocessing the input data: {e}") - return None + logging.fatal( + f"An error occurred while preprocessing the input data: {e}") + def main(argv): + del argv try: - download_file(url=Commerce_NTIA_URL, - output_folder=INPUT_DIR, - unzip=False, - headers= None, - tries= 3, - delay= 5, - backoff= 2) + success = download_file( + url=Commerce_NTIA_URL, + output_folder=INPUT_DIR, + unzip=False, + headers=HEADERS, + tries=3, + delay=5, + backoff=2, + ) + if not success or not os.path.exists(INPUT_FILE) or os.path.getsize( + INPUT_FILE) == 0: + logging.fatal( + "Failed to download Commerce_NTIA file or file is empty.") except Exception as e: logging.fatal(f"Failed to download Commerce_NTIA file: {e}") + preprocess_data() + if __name__ == "__main__": app.run(main) diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/validation_config.json b/statvar_imports/ntia_internet_use_survey/commerce_ntia/validation_config.json new file mode 100644 index 0000000000..de76934934 --- /dev/null +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/validation_config.json @@ -0,0 +1,31 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted records for the entire import is within threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.1 + } + }, + { + "rule_id": "check_active_survey_wave_count", + "description": "Checks that all active StatVars (1935 in input0, 227 in input1) have MaxDate >= 2023-11.", + "validator": "SQL_VALIDATOR", + "params": { + "query": "SELECT COUNT(*) AS total_svs, SUM(CASE WHEN MaxDate >= '2023-11' THEN 1 ELSE 0 END) AS active_svs FROM stats", + "condition": "active_svs >= CASE WHEN total_svs > 500 THEN 1935 ELSE 227 END" + } + }, + { + "rule_id": "check_statvar_min_max_date", + "description": "Checks that no StatVar regresses below the oldest historical wave (2013-07).", + "validator": "SQL_VALIDATOR", + "params": { + "query": "SELECT StatVar, MaxDate FROM stats", + "condition": "MaxDate >= '2013-07'" + } + } + ] +} \ No newline at end of file