From 7428ef6cb18cf74ea72cfdec6c0e3de8327ed0a3 Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Wed, 9 Sep 2026 08:12:37 +0000 Subject: [PATCH 01/11] ny_brfss_health_indicators --- .../ny_brfss_health_indicators/README.md | 105 ++++++++++++ .../ny_brfss_health_indicators/download.py | 160 ++++++++++++++++++ .../ny_brfss_health_indicators/manifest.json | 29 ++++ .../ny_brfss_health_indicators_metadata.csv | 5 + .../ny_brfss_health_indicators_pv_map.csv | 148 ++++++++++++++++ .../test_data/sample_expected_output.csv | 31 ++++ .../test_data/sample_expected_output.tmcf | 7 + .../sample_expected_output_stat_vars.mcf | 38 +++++ ...ample_expected_output_stat_vars_schema.mcf | 15 ++ .../test_data/sample_input.csv | 6 + .../validation_config.json | 13 ++ 11 files changed, 557 insertions(+) create mode 100644 statvar_imports/us_newyork/ny_brfss_health_indicators/README.md create mode 100644 statvar_imports/us_newyork/ny_brfss_health_indicators/download.py create mode 100644 statvar_imports/us_newyork/ny_brfss_health_indicators/manifest.json create mode 100644 statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_metadata.csv create mode 100644 statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_pv_map.csv create mode 100644 statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output.csv create mode 100644 statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output.tmcf create mode 100644 statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output_stat_vars.mcf create mode 100644 statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output_stat_vars_schema.mcf create mode 100644 statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_input.csv create mode 100644 statvar_imports/us_newyork/ny_brfss_health_indicators/validation_config.json diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md b/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md new file mode 100644 index 0000000000..749489a46e --- /dev/null +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md @@ -0,0 +1,105 @@ +# NewYork_BRFSS_Health_Indicators + +## 1. Import Overview + +This import ingests health indicator prevalence rates across **all 62 counties** in New York State and New York City. The dataset is sourced directly from the **New York State Department of Health (NYSDOH) Behavioral Risk Factor Surveillance System (eBRFSS)**. + +* **Import Directory**: `statvar_imports/us_newyork/ny_brfss_health_indicators/` +* **Import Name**: `NewYork_BRFSS_Health_Indicators` +* **Dataset Landing Page**: [https://health.data.ny.gov/Health/Behavioral-Risk-Factor-Surveillance-System-BRFSS-H/jsy7-eb4n/about_data](https://health.data.ny.gov/Health/Behavioral-Risk-Factor-Surveillance-System-BRFSS-H/jsy7-eb4n/about_data) +* **Socrata API Endpoint**: `https://health.data.ny.gov/resource/jsy7-eb4n.json` +* **Geographic Coverage**: **All 62 NY Counties** (`geoId/36001` through `geoId/36123`) and **New York City** (`geoId/3651000`). +* **Temporal Coverage**: 2014, 2016, 2018, 2021, and 2024 survey releases. +* **Unit & Measurement Method**: Direct survey unadjusted crude prevalence percentages (`Percent`, `scalingFactor: 100`). + +--- + +## 2. Health Indicators & Statistical Variables + +The import supports **all 75 health indicators** in the NYSDOH dataset across 6 primary health domains: + +| Health Domain | Indicators | Key Topics Covered | +|---|:---:|---| +| **Prevent Chronic Diseases** | 42 | Arthritis, Asthma, Cardiovascular Disease, COPD, Diabetes, High Blood Pressure, Obesity, Overweight, Current Smoking (all age/income/disability subsets), e-Cigarettes, Breast/Cervical/Colorectal/PSA Screenings, Physical Activity, Fast Food, Sugary Drinks, Fruits/Vegetables | +| **Improve Health Status & Reduce Disparities** | 13 | Disability (6-question ACA standard), Limitation Status, Health Care Coverage, Personal Health Care Provider, Annual Checkup, Dental Visit, Medical Care Cost Barrier, Poor General & Physical Health, Food Insecurity, Food Security, Housing Insecurity | +| **Promote Mental Health & Substance Abuse** | 9 | Depressive Disorder, Poor Mental Health ($\ge 14$ days), Binge Drinking, Heavy Drinking, DWI, Cannabis Use, Medical Cannabis Use, Smoking with Poor Mental Health, Adverse Childhood Experiences (ACEs) | +| **Prevent HIV/STDs, Vaccine-Preventable Diseases & HAIs** | 5 | Influenza Immunization (Ages 18+, Ages 65+), Pneumococcal Immunization (Ages 65+), Hepatitis C (HCV) Testing (All Adults, Ages 47-68) | +| **Promote Healthy Women, Infants, and Children** | 5 | Female Health Care Coverage (18-64), Female Routine Checkups (18-44, 18-64), Female Dental Visits (18-44), Health Care Provider Discussed Healthy Pregnancy | +| **Promote a Healthy and Safe Environment** | 1 | Walkable Neighborhoods Suitable for Physical Activity | +| **Total** | **75** | **13,479 Observations** | + +### Statistical Variable Resolution +* **Canonical StatVars (12)**: Already established in the Data Commons Knowledge Graph (e.g. `Percent_Person_WithDiabetes`, `Percent_Person_WithAsthma`, `Percent_Person_WithArthritis`, `Percent_Person_WithChronicObstructivePulmonaryDisease`, `Percent_Person_WithHighBloodPressure`, `Percent_Person_WithHighCholesterol`, `Percent_Person_Obesity`, `Percent_Person_18OrMoreYears_WithDepression`, `Percent_Person_Smoking`, `Percent_Person_BingeDrinking`, `Percent_Person_18OrMoreYears_WithPoorGeneralHealth`, `Percent_Person_WithMentalHealthNotGood`, `Percent_Person_WithPhysicalHealthNotGood`). +* **Provisional StatVars (63)**: Synthesized with formal property-value graphs and emitted into `output_files/ny_brfss_health_indicators_output_stat_vars.mcf`. +* **Supporting Schema Nodes (58)**: Provisional enums, properties, and types emitted into `output_files/ny_brfss_health_indicators_output_stat_vars_schema.mcf`. + +--- + +## 3. Directory Layout + +``` +statvar_imports/us_newyork/ny_brfss_health_indicators/ +├── README.md # Comprehensive import documentation +├── manifest.json # Data Commons import automation manifest +├── validation_config.json # Import validation rules (historical deletion threshold) +├── ny_brfss_health_indicators_metadata.csv # Processor configuration metadata +├── ny_brfss_health_indicators_pv_map.csv # Property-value mapping for all 75 StatVars & 62 counties +├── schema.mcf # Combined local test schema (provisional StatVars & enums) +├── download.py # Multi-year Socrata downloader +├── download_test.py # Downloader unit test suite +├── input_files/ # Multi-year source datasets from NYSDOH API +│ └── ny_brfss_health_indicators_raw.csv # Complete raw unpivoted dataset (all 17,700 records) +├── output_files/ # Generated Data Commons artifacts +│ ├── ny_brfss_health_indicators_output.csv # Cleaned StatVarObservations (13,479 rows) +│ ├── ny_brfss_health_indicators_output.tmcf # Template MCF mapping file +│ ├── ny_brfss_health_indicators_output_stat_vars.mcf # 63 Provisional StatVar nodes +│ └── ny_brfss_health_indicators_output_stat_vars_schema.mcf # 58 Provisional schema definitions +└── counters/ # Processor execution statistics + └── ny_brfss_health_indicators_counters.csv +``` + +--- + +## 4. Execution Workflow + +### Step 1: Download Raw Health Data +To fetch the full historical survey data: +```bash +python3 download.py +``` +This queries the NYSDOH Socrata API for all 17,700 records across all 75 indicators and 5 survey waves, writing directly to `input_files/ny_brfss_health_indicators_raw.csv`. + +### Step 2: Generate StatVar Observations & TMCF +Run `stat_var_processor.py` directly on the raw dataset from within the import directory: +```bash +python3 ../../../tools/statvar_importer/stat_var_processor.py \ + --input_data="input_files/ny_brfss_health_indicators_raw.csv" \ + --pv_map=ny_brfss_health_indicators_pv_map.csv \ + --config_file=ny_brfss_health_indicators_metadata.csv \ + --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_path=output_files/ny_brfss_health_indicators_output \ + --output_counters=counters/ny_brfss_health_indicators_counters.csv +``` + +### Step 3: Run Unit Tests +```bash +python3 -m unittest download_test.py +``` + +### Step 4: Validate with Data Commons Import Tool +```bash +java -jar ~/Downloads/import_tools_import-tool.jar lint \ + output_files/ny_brfss_health_indicators_output.csv \ + output_files/ny_brfss_health_indicators_output.tmcf \ + schema.mcf +``` + +--- + +## 5. Verification Results + +* **Observations Generated**: **13,479** `StatVarObservation` records. +* **Geographies Resolved**: **63 unique entities** (all 62 NY counties `geoId/36001` - `geoId/36123` plus New York City `geoId/3651000`). +* **Non-County Regions Dropped**: **2,831 records** (sub-state DSRIP regions, Rest of State, and Statewide rows cleanly dropped by PV map). +* **Statistical Variables Generated**: **75 unique variables** (12 matched to existing canonical DCIDs, 63 provisional StatVars, 58 schema enums/properties). +* **Linter Status**: **0 fatal, 0 errors, 0 missing references** (`NumRowSuccesses: 13,479 / 13,479`). diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py b/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py new file mode 100644 index 0000000000..cd7009676b --- /dev/null +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py @@ -0,0 +1,160 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Automated Multi-Year Downloader for NYS BRFSS Health Indicators by County and Region. + +Downloads live county and region level health outcome indicators for New York State +from the NYSDOH Socrata Open Data API (jsy7-eb4n) across all survey years (2014, 2016, +2018, 2021, 2024), covering all 62 counties, 11 DSRIP regions, NYC, Rest of State, +and Statewide. +""" + +import os +import sys +from absl import app +from absl import flags +from absl import logging +import pandas as pd +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +FLAGS = flags.FLAGS + +flags.DEFINE_string( + 'output_dir', + 'input_files', + 'Directory where downloaded input CSV files will be saved.', +) +flags.DEFINE_string( + 'endpoint', + 'https://health.data.ny.gov/resource/jsy7-eb4n.json', + 'Health Data NY BRFSS Socrata API endpoint.', +) + +# Mapping of NY DOH short indicator names to PV map column names +INDICATOR_COLUMN_MAP = { + 'Diabetes': 'Percentage of adults with diabetes', + 'Prediabetes': 'Percentage of adults with prediabetes', + 'Asthma': 'Percentage of adults with asthma', + 'Arthritis': 'Percentage of adults with arthritis', + 'Cardiovascular Disease': 'Percentage of adults with cardiovascular disease', + 'COPD': 'Percentage of adults with COPD', + 'High Blood Pressure': 'Percentage of adults with high blood pressure', + 'Elevated Cholesterol': 'Percentage of adults with elevated cholesterol', + 'Obesity': 'Percentage of adults with obesity', + 'Overweight or Obese': 'Percentage of adults overweight or obese', + 'Depressive Disorder': 'Percentage of adults with depression', +} + + +def create_session() -> requests.Session: + """Creates a requests session configured with retries and connection pooling.""" + session = requests.Session() + retries = Retry( + total=5, + backoff_factor=1.5, + status_forcelist=[429, 500, 502, 503, 504], + raise_on_status=False, + ) + adapter = HTTPAdapter(max_retries=retries) + session.mount('https://', adapter) + session.mount('http://', adapter) + session.headers.update({ + 'User-Agent': 'Mozilla/5.0 (DataCommons Ingestion; +https://datacommons.org)', + 'Accept': 'application/json, text/csv, */*', + }) + return session + + +def normalize_year(raw_year: str) -> str: + """Normalizes survey year interval to a 4-digit numeric observation year.""" + raw_year = str(raw_year).strip() + if '-' in raw_year: + parts = raw_year.split('-') + start = parts[0].strip() + end = parts[1].strip() + if len(end) == 2 and len(start) == 4: + return f'{start[:2]}{end}' + if len(end) == 4: + return end + return end + return raw_year + + +def atomic_to_csv(df: pd.DataFrame, target_path: str) -> None: + """Writes a DataFrame to a CSV file atomically via a temporary file.""" + temp_path = f'{target_path}.tmp' + df.to_csv(temp_path, index=False) + os.replace(temp_path, target_path) + + +def download_health_indicators(endpoint: str, output_dir: str) -> tuple[int, list[str]]: + """Downloads all health indicators for all NY counties and regions.""" + session = create_session() + os.makedirs(output_dir, exist_ok=True) + + records = [] + offset = 0 + limit = 50000 + + while True: + params = { + '$limit': limit, + '$offset': offset, + } + logging.info('GET %s with params %s', endpoint, params) + resp = session.get(endpoint, params=params, timeout=60) + if resp.status_code != 200: + logging.fatal( + 'Health Data NY API returned HTTP %d: %s', resp.status_code, resp.text[:200] + ) + return 0, [] + + chunk = resp.json() + if not chunk: + break + records.extend(chunk) + logging.info('Fetched %d records (offset %d).', len(chunk), offset) + if len(chunk) < limit: + break + offset += limit + + if not records: + logging.fatal('Health Data NY API returned 0 records.') + return 0, [] + + logging.info('Received %d total raw records from Health Data NY API.', len(records)) + raw_df = pd.DataFrame(records) + + # Save complete raw unpivoted dataset (all 17,000+ API records) + raw_file = os.path.join(output_dir, 'ny_brfss_health_indicators_raw.csv') + atomic_to_csv(raw_df, raw_file) + saved_files = [raw_file] + logging.info('Wrote raw unpivoted dataset (%d records) -> %s', len(raw_df), raw_file) + + return len(raw_df), saved_files + + +def main(_): + script_dir = os.path.dirname(os.path.abspath(__file__)) + output_dir = ( + FLAGS.output_dir + if os.path.isabs(FLAGS.output_dir) + else os.path.join(script_dir, FLAGS.output_dir) + ) + download_health_indicators(FLAGS.endpoint, output_dir) + + +if __name__ == '__main__': + app.run(main) diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/manifest.json b/statvar_imports/us_newyork/ny_brfss_health_indicators/manifest.json new file mode 100644 index 0000000000..d0d477f7a3 --- /dev/null +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/manifest.json @@ -0,0 +1,29 @@ +{ + "import_specifications": [ + { + "import_name": "NewYork_BRFSS_Health_Indicators", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://health.data.ny.gov/Health/Behavioral-Risk-Factor-Surveillance-System-BRFSS-H/jsy7-eb4n/about_data", + "provenance_description": "The Behavioral Risk Factor Surveillance System (BRFSS) Health Indicators dataset from the New York State Department of Health provides annual estimates of chronic disease prevalence across all counties in New York.", + "scripts": [ + "download.py", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=\"input_files/ny_brfss_health_indicators_raw.csv\" --pv_map=ny_brfss_health_indicators_pv_map.csv --config_file=ny_brfss_health_indicators_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ny_brfss_health_indicators_output --output_counters=counters/ny_brfss_health_indicators_counters.csv" + ], + "import_inputs": [ + { + "template_mcf": "output_files/ny_brfss_health_indicators_output.tmcf", + "cleaned_csv": "output_files/ny_brfss_health_indicators_output.csv", + "node_mcf": "output_files/*.mcf" + } + ], + "source_files": [ + "input_files/*.csv", + "counters/*.csv" + ], + "cron_schedule": "0 0 1 8 *", + "validation_config_file": "validation_config.json" + } + ] +} \ No newline at end of file diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_metadata.csv b/statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_metadata.csv new file mode 100644 index 0000000000..9158c3e7f2 --- /dev/null +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_metadata.csv @@ -0,0 +1,5 @@ +parameter,value +url,https://health.data.ny.gov/Health/Behavioral-Risk-Factor-Surveillance-System-BRFSS-H/jsy7-eb4n/about_data +output_columns,"observationAbout,observationDate,value,variableMeasured" +header_rows,1 +mapped_columns,4 diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_pv_map.csv b/statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_pv_map.csv new file mode 100644 index 0000000000..40752a4ebe --- /dev/null +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_pv_map.csv @@ -0,0 +1,148 @@ +key,p1,v1,p2,v2,p3,v3,p4,v4,p5,v5,p6,v6,p7,v7,p8,v8,p9,v9,p10,v10,p11,v11,p12,v12,p13,v13 +years,observationDate,{@Data} +2013-14,observationDate,2014 +region_county,observationAbout,{@Data} +unadjusted_rate,value,{Number} +DSRIP region,#IgnoreRow,True +NYS Exclusive of NYC,#IgnoreRow,True +Statewide,#IgnoreRow,True +REGION: New York City,#IgnoreRow,True +Diabetes,populationType,Person,measuredProperty,percent,healthOutcome,Diabetes,unit,Percent +Prediabetes,populationType,Person,measuredProperty,percent,healthOutcome,Prediabetes,unit,Percent +Asthma,populationType,Person,measuredProperty,percent,healthOutcome,Asthma,unit,Percent +Arthritis,populationType,Person,measuredProperty,percent,healthOutcome,Arthritis,unit,Percent +Cardiovascular Disease,populationType,Person,measuredProperty,percent,healthOutcome,CardiovascularDisease,unit,Percent +COPD,populationType,Person,measuredProperty,percent,healthOutcome,ChronicObstructivePulmonaryDisease,unit,Percent +High Blood Pressure,populationType,Person,measuredProperty,percent,healthOutcome,HighBloodPressure,unit,Percent +Elevated Cholesterol,populationType,Person,measuredProperty,percent,healthOutcome,HighCholesterol,unit,Percent +Obesity,dcid,Percent_Person_Obesity,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthBehavior,Obesity,unit,Percent +Overweight or Obese,populationType,Person,measuredProperty,percent,healthOutcome,OverweightOrObese,unit,Percent +Depressive Disorder,dcid,Percent_Person_18OrMoreYears_WithDepression,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthOutcome,Depression,unit,Percent +2+ Adverse Childhood Experiences,populationType,Person,measuredProperty,percent,healthStatus,TwoOrMoreAdverseChildhoodExperiences,unit,Percent +Active Transportation,populationType,Person,measuredProperty,percent,healthBehavior,ActiveTransportation,unit,Percent +Asthma Ever,populationType,Person,measuredProperty,percent,healthOutcome,AsthmaEver,unit,Percent +Binge Drinking,dcid,Percent_Person_BingeDrinking,populationType,Person,measuredProperty,percent,healthBehavior,BingeDrinking,unit,Percent +Binge or Heavy Drinking,populationType,Person,measuredProperty,percent,healthBehavior,BingeOrHeavyDrinking,unit,Percent +Breast Cancer Screening,dcid,Percent_Person_50To74Years_Female_ReceivedMammography,populationType,Person,gender,Female,age,Years50To74,measuredProperty,percent,healthCareUtilization,Mammography,unit,Percent +Breast Cancer Screening Low Income 50-74,populationType,Person,gender,Female,age,Years50To74,income,USDollar0To24999,measuredProperty,percent,healthCareUtilization,Mammography,unit,Percent +Cardiovascular Disease Disability,populationType,Person,healthStatus,WithDisability,measuredProperty,percent,healthOutcome,CardiovascularDisease,unit,Percent +Cervical Cancer Screening,dcid,Percent_Person_21To65Years_Female_ReceivedCervicalCancerScreening,populationType,Person,gender,Female,age,Years21To65,measuredProperty,percent,healthCareUtilization,CervicalCancerScreening,unit,Percent +Cervical Cancer Screening (2012 guidelines),populationType,Person,gender,Female,age,Years21To65,measuredProperty,percent,healthCareUtilization,CervicalCancerScreening2012Guidelines,unit,Percent +Cervical Cancer Screening (2012 guidelines) 21-29,populationType,Person,gender,Female,age,Years21To29,measuredProperty,percent,healthCareUtilization,CervicalCancerScreening2012Guidelines,unit,Percent +Cervical Cancer Screening (pre-2012 guidelines),populationType,Person,gender,Female,age,Years21To65,measuredProperty,percent,healthCareUtilization,CervicalCancerScreeningPre2012Guidelines,unit,Percent +Cervical Cancer Screening (pre-2012 guidelines) Low Income,populationType,Person,gender,Female,age,Years21To65,income,USDollar0To24999,measuredProperty,percent,healthCareUtilization,CervicalCancerScreeningPre2012Guidelines,unit,Percent +Cholesterol Checked,dcid,Percent_Person_ReceivedCholesterolScreening,populationType,Person,measuredProperty,percent,healthCareUtilization,CholesterolScreening,unit,Percent +Chronic Disease Self-Management,populationType,Person,measuredProperty,percent,healthBehavior,ChronicDiseaseSelfManagement,unit,Percent +Colorectal Cancer Screening,dcid,Percent_Person_50To75Years_ReceivedColorectalCancerScreening,populationType,Person,age,Years50To75,measuredProperty,percent,healthCareUtilization,ColorectalCancerScreening,unit,Percent +Colorectal Cancer Screening 50-64,populationType,Person,age,Years50To64,measuredProperty,percent,healthCareUtilization,ColorectalCancerScreening,unit,Percent +Colorectal Cancer Screening Low Income,populationType,Person,age,Years50To75,income,USDollar0To24999,measuredProperty,percent,healthCareUtilization,ColorectalCancerScreening,unit,Percent +Current Cannabis Use,populationType,Person,measuredProperty,percent,healthBehavior,CannabisUse,unit,Percent +Current Medical Cannabis Use,populationType,Person,measuredProperty,percent,healthBehavior,MedicalCannabisUse,unit,Percent +Current Smoking,dcid,Percent_Person_Smoking,populationType,Person,measuredProperty,percent,healthBehavior,SmokingStatus_CurrentSmoker,unit,Percent +Current Smoking 18-24,populationType,Person,age,Years18To24,measuredProperty,percent,healthBehavior,SmokingStatus_CurrentSmoker,unit,Percent +Current Smoking Disability,populationType,Person,healthStatus,WithDisability,measuredProperty,percent,healthBehavior,SmokingStatus_CurrentSmoker,unit,Percent +Current Smoking Low Income,populationType,Person,income,USDollar0To24999,measuredProperty,percent,healthBehavior,SmokingStatus_CurrentSmoker,unit,Percent +Current Smoking Poor Mental Health,populationType,Person,healthStatus,PoorMentalHealth,measuredProperty,percent,healthBehavior,SmokingStatus_CurrentSmoker,unit,Percent +DWI,populationType,Person,measuredProperty,percent,healthBehavior,DrivingWhileIntoxicated,unit,Percent +Diabetes and Prediabetes Testing,populationType,Person,measuredProperty,percent,healthCareUtilization,DiabetesOrPrediabetesTesting,unit,Percent +Disability,dcid,Percent_Person_18OrMoreYears_WithAnyDisability,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthStatus,WithDisability,unit,Percent +Fall within Past Year 65+,populationType,Person,age,Years65Onwards,measuredProperty,percent,healthOutcome,FallPastYear,unit,Percent +Falls Resulting in Injury 65+,populationType,Person,age,Years65Onwards,measuredProperty,percent,healthOutcome,FallWithInjuryPastYear,unit,Percent +Fast Food Consumption,populationType,Person,measuredProperty,percent,healthBehavior,FastFoodConsumption3OrMoreTimesPerWeek,unit,Percent +Flu-Shot,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthCareUtilization,InfluenzaVaccinePastYear,unit,Percent +Flu-Shot 65+,populationType,Person,age,Years65Onwards,measuredProperty,percent,healthCareUtilization,InfluenzaVaccinePastYear,unit,Percent +Food Insecurity,populationType,Person,measuredProperty,percent,socialSecurityStatus,FoodInsecurityPast12Months,unit,Percent +Food Security,populationType,Person,measuredProperty,percent,socialSecurityStatus,FoodSecurityPast12Months,unit,Percent +HCV Ever Tested,populationType,Person,measuredProperty,percent,healthCareUtilization,HepatitisCTestingEver,unit,Percent +HCV Ever Tested 47-68,populationType,Person,age,Years47To68,measuredProperty,percent,healthCareUtilization,HepatitisCTestingEver,unit,Percent +Health Care Coverage,populationType,Person,age,Years18To64,measuredProperty,percent,healthInsurance,WithHealthInsurance,unit,Percent +Health Care Coverage Female 18-64,populationType,Person,gender,Female,age,Years18To64,measuredProperty,percent,healthInsurance,WithHealthInsurance,unit,Percent +Health Care Provider,populationType,Person,measuredProperty,percent,healthCareProviderStatus,HasPersonalHealthCareProvider,unit,Percent +High Blood Pressure Taking Medication,dcid,Percent_Person_18OrMoreYears_WithHighBloodPressure_ReceivedTakingBloodPressureMedication,populationType,Person,age,Years18Onwards,measuredProperty,percent,medicalCondition,HighBloodPressureMedication,unit,Percent +Housing Insecurity,populationType,Person,measuredProperty,percent,socialSecurityStatus,HousingInsecurityPast12Months,unit,Percent +Housing or Utility Insecurity,populationType,Person,measuredProperty,percent,socialSecurityStatus,HousingOrUtilityInsecurityPast12Months,unit,Percent +Leisure-Time Physical Activity,populationType,Person,measuredProperty,percent,healthBehavior,LeisureTimePhysicalActivity,unit,Percent +Limitation Status,populationType,Person,measuredProperty,percent,healthStatus,WithLimitation,unit,Percent +No Fruits or Vegetables,populationType,Person,measuredProperty,percent,healthBehavior,ConsumesNoFruitsOrVegetables,unit,Percent +No Medical Care Due to Cost,populationType,Person,measuredProperty,percent,healthCareAccessStatus,DidNotReceiveCareDueToCost,unit,Percent +Obesity Disability,populationType,Person,healthStatus,WithDisability,measuredProperty,percent,healthBehavior,Obesity,unit,Percent +Obesity Limitation Status,populationType,Person,healthStatus,WithLimitation,measuredProperty,percent,healthBehavior,Obesity,unit,Percent +Obesity Low Income,populationType,Person,income,USDollar0To24999,measuredProperty,percent,healthBehavior,Obesity,unit,Percent +PSA Screening,populationType,Person,gender,Male,age,Years55To69,measuredProperty,percent,healthCareUtilization,ProstateSpecificAntigenScreeningSharedDecision,unit,Percent +Pneumonia Shot 65+,populationType,Person,age,Years65Onwards,measuredProperty,percent,healthCareUtilization,PneumococcalVaccineEver,unit,Percent +Poor General Health,dcid,Percent_Person_18OrMoreYears_WithPoorGeneralHealth,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthOutcome,PoorGeneralHealth,unit,Percent +Poor Mental Health,dcid,Percent_Person_WithMentalHealthNotGood,populationType,Person,measuredProperty,percent,healthOutcome,MentalHealthNotGood,unit,Percent +Poor Physical Health,dcid,Percent_Person_WithPhysicalHealthNotGood,populationType,Person,measuredProperty,percent,healthOutcome,PhysicalHealthNotGood,unit,Percent +Provider Discussed Healthy Pregnancy,populationType,Person,gender,Female,age,Years18To44,measuredProperty,percent,healthCareUtilization,ProviderDiscussedHealthyPregnancy,unit,Percent +Recent Checkup,dcid,Percent_Person_ReceivedAnnualCheckup,populationType,Person,age,Years18To64,measuredProperty,percent,healthCareUtilization,RoutineCheckupWithinPastYear,unit,Percent +Recent Checkup Female 18-44,populationType,Person,gender,Female,age,Years18To44,measuredProperty,percent,healthCareUtilization,RoutineCheckupWithinPastYear,unit,Percent +Recent Checkup Female 18-64,populationType,Person,gender,Female,age,Years18To64,measuredProperty,percent,healthCareUtilization,RoutineCheckupWithinPastYear,unit,Percent +Recent Dental Visit,dcid,Percent_Person_ReceivedDentalVisit,populationType,Person,measuredProperty,percent,healthCareUtilization,DentalVisitWithinPastYear,unit,Percent +Recent Dental Visit Female 18-44,populationType,Person,gender,Female,age,Years18To44,measuredProperty,percent,healthCareUtilization,DentalVisitWithinPastYear,unit,Percent +Sugar-Sweetened Beverages,populationType,Person,measuredProperty,percent,healthBehavior,DailySugarSweetenedBeverages,unit,Percent +Walkable Neighborhood,populationType,Person,measuredProperty,percent,environmentStatus,WalkableNeighborhood,unit,Percent +e-Cigarettes,populationType,Person,measuredProperty,percent,healthBehavior,ElectronicCigaretteUse,unit,Percent +Albany,observationAbout,geoId/36001 +Allegany,observationAbout,geoId/36003 +Bronx,observationAbout,geoId/36005 +Broome,observationAbout,geoId/36007 +Cattaraugus,observationAbout,geoId/36009 +Cayuga,observationAbout,geoId/36011 +Chautauqua,observationAbout,geoId/36013 +Chemung,observationAbout,geoId/36015 +Chenango,observationAbout,geoId/36017 +Clinton,observationAbout,geoId/36019 +Columbia,observationAbout,geoId/36021 +Cortland,observationAbout,geoId/36023 +Delaware,observationAbout,geoId/36025 +Dutchess,observationAbout,geoId/36027 +Erie,observationAbout,geoId/36029 +Essex,observationAbout,geoId/36031 +Franklin,observationAbout,geoId/36033 +Fulton,observationAbout,geoId/36035 +Genesee,observationAbout,geoId/36037 +Greene,observationAbout,geoId/36039 +Hamilton,observationAbout,geoId/36041 +Herkimer,observationAbout,geoId/36043 +Jefferson,observationAbout,geoId/36045 +Kings,observationAbout,geoId/36047 +Lewis,observationAbout,geoId/36049 +Livingston,observationAbout,geoId/36051 +Madison,observationAbout,geoId/36053 +Monroe,observationAbout,geoId/36055 +Montgomery,observationAbout,geoId/36057 +Nassau,observationAbout,geoId/36059 +New York City,observationAbout,geoId/3651000 +New York,observationAbout,geoId/36061 +Niagara,observationAbout,geoId/36063 +Oneida,observationAbout,geoId/36065 +Onondaga,observationAbout,geoId/36067 +Ontario,observationAbout,geoId/36069 +Orange,observationAbout,geoId/36071 +Orleans,observationAbout,geoId/36073 +Oswego,observationAbout,geoId/36075 +Otsego,observationAbout,geoId/36077 +Putnam,observationAbout,geoId/36079 +Queens,observationAbout,geoId/36081 +Rensselaer,observationAbout,geoId/36083 +Richmond,observationAbout,geoId/36085 +Rockland,observationAbout,geoId/36087 +Saratoga,observationAbout,geoId/36091 +Schenectady,observationAbout,geoId/36093 +Schoharie,observationAbout,geoId/36095 +Schuyler,observationAbout,geoId/36097 +Seneca,observationAbout,geoId/36099 +St Lawrence,observationAbout,geoId/36089 +St. Lawrence,observationAbout,geoId/36089 +Steuben,observationAbout,geoId/36101 +Suffolk,observationAbout,geoId/36103 +Sullivan,observationAbout,geoId/36105 +Tioga,observationAbout,geoId/36107 +Tompkins,observationAbout,geoId/36109 +Ulster,observationAbout,geoId/36111 +Warren,observationAbout,geoId/36113 +Washington,observationAbout,geoId/36115 +Wayne,observationAbout,geoId/36117 +Westchester,observationAbout,geoId/36119 +Wyoming,observationAbout,geoId/36121 +Yates,observationAbout,geoId/36123 diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output.csv b/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output.csv new file mode 100644 index 0000000000..e71b82b96c --- /dev/null +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output.csv @@ -0,0 +1,31 @@ +observationAbout,observationDate,value,variableMeasured +geoId/36001,2021,9.2,dcid:Percent_Person_WithDiabetes +geoId/36001,2021,11.7,dcid:Percent_Person_WithPrediabetes +geoId/36001,2021,11.3,dcid:Percent_Person_WithAsthma +geoId/36001,2021,23.7,dcid:Percent_Person_WithArthritis +geoId/36001,2021,7.4,dcid:Percent_Person_WithCardiovascularDisease +geoId/36001,2021,6.7,dcid:Percent_Person_WithChronicObstructivePulmonaryDisease +geoId/36001,2021,29.6,dcid:Percent_Person_WithHighBloodPressure +geoId/36001,2021,30.1,dcid:Percent_Person_WithObesity +geoId/36001,2021,61.9,dcid:Percent_Person_WithOverweightOrObese +geoId/36001,2021,21.2,dcid:Percent_Person_Years18Onwards_WithDepression +geoId/36005,2021,13.6,dcid:Percent_Person_WithDiabetes +geoId/36005,2021,15.2,dcid:Percent_Person_WithPrediabetes +geoId/36005,2021,12.4,dcid:Percent_Person_WithAsthma +geoId/36005,2021,19,dcid:Percent_Person_WithArthritis +geoId/36005,2021,6.6,dcid:Percent_Person_WithCardiovascularDisease +geoId/36005,2021,4.4,dcid:Percent_Person_WithChronicObstructivePulmonaryDisease +geoId/36005,2021,33,dcid:Percent_Person_WithHighBloodPressure +geoId/36005,2021,34.5,dcid:Percent_Person_WithObesity +geoId/36005,2021,70.2,dcid:Percent_Person_WithOverweightOrObese +geoId/36005,2021,16.7,dcid:Percent_Person_Years18Onwards_WithDepression +geoId/3651000,2021,12,dcid:Percent_Person_WithDiabetes +geoId/3651000,2021,12.9,dcid:Percent_Person_WithPrediabetes +geoId/3651000,2021,9.1,dcid:Percent_Person_WithAsthma +geoId/3651000,2021,20.2,dcid:Percent_Person_WithArthritis +geoId/3651000,2021,6.3,dcid:Percent_Person_WithCardiovascularDisease +geoId/3651000,2021,4.1,dcid:Percent_Person_WithChronicObstructivePulmonaryDisease +geoId/3651000,2021,28.8,dcid:Percent_Person_WithHighBloodPressure +geoId/3651000,2021,25.7,dcid:Percent_Person_WithObesity +geoId/3651000,2021,59.5,dcid:Percent_Person_WithOverweightOrObese +geoId/3651000,2021,15.6,dcid:Percent_Person_Years18Onwards_WithDepression diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output.tmcf b/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output.tmcf new file mode 100644 index 0000000000..28fa13adbb --- /dev/null +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output.tmcf @@ -0,0 +1,7 @@ +Node: E:sample_expected_output->E0 +observationAbout: C:sample_expected_output->observationAbout +observationDate: C:sample_expected_output->observationDate +value: C:sample_expected_output->value +variableMeasured: C:sample_expected_output->variableMeasured +typeOf: dcs:StatVarObservation +unit: Percent diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output_stat_vars.mcf b/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output_stat_vars.mcf new file mode 100644 index 0000000000..e6a1b1c452 --- /dev/null +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output_stat_vars.mcf @@ -0,0 +1,38 @@ +# Auto generated using command: "tools/statvar_importer/stat_var_processor.py --input_data=statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_input.csv --pv_map=statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_pv_map.csv --config_file=statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_metadata.csv --output_path=statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output" on 2026-09-03 14:13:29.017132 + +Node: dcid:Percent_Person_WithCardiovascularDisease +typeOf: dcid:StatisticalVariable +populationType: dcid:Person +measuredProperty: dcid:percent +statType: dcid:measuredValue +healthOutcome: dcid:CardiovascularDisease + +Node: dcid:Percent_Person_WithObesity +typeOf: dcid:StatisticalVariable +populationType: dcid:Person +measuredProperty: dcid:percent +statType: dcid:measuredValue +healthOutcome: dcid:Obesity + +Node: dcid:Percent_Person_WithOverweightOrObese +typeOf: dcid:StatisticalVariable +populationType: dcid:Person +measuredProperty: dcid:percent +statType: dcid:measuredValue +healthOutcome: dcid:OverweightOrObese + +Node: dcid:Percent_Person_WithPrediabetes +typeOf: dcid:StatisticalVariable +populationType: dcid:Person +measuredProperty: dcid:percent +statType: dcid:measuredValue +healthOutcome: dcid:Prediabetes + +Node: dcid:Percent_Person_Years18Onwards_WithDepression +typeOf: dcid:StatisticalVariable +populationType: dcid:Person +measuredProperty: dcid:percent +statType: dcid:measuredValue +age: dcid:Years18Onwards +healthOutcome: dcid:Depression + diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output_stat_vars_schema.mcf b/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output_stat_vars_schema.mcf new file mode 100644 index 0000000000..33e0428767 --- /dev/null +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output_stat_vars_schema.mcf @@ -0,0 +1,15 @@ +Node: dcid:Prediabetes +typeOf: dcid:HealthOutcomeEnum +name: "Prediabetes" +isProvisional: dcs:True + +Node: dcid:healthOutcome +typeOf: dcs:Property +name: "healthOutcome" +rangeIncludes: dcid:MedicalConditionIncidentTypeEnum,dcid:BodyMassIndexClassificationEnum + +Node: dcid:OverweightOrObese +typeOf: dcid:HealthOutcomeEnum +name: "Overweight Or Obese" +isProvisional: dcs:True + diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_input.csv b/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_input.csv new file mode 100644 index 0000000000..0ad2020c07 --- /dev/null +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_input.csv @@ -0,0 +1,6 @@ +Location,Year,Percentage of adults with diabetes,Percentage of adults with prediabetes,Percentage of adults with asthma,Percentage of adults with arthritis,Percentage of adults with cardiovascular disease,Percentage of adults with COPD,Percentage of adults with high blood pressure,Percentage of adults with elevated cholesterol,Percentage of adults with obesity,Percentage of adults overweight or obese,Percentage of adults with depression +Albany,2021,9.2,11.7,11.3,23.7,7.4,6.7,29.6,,30.1,61.9,21.2 +Bronx,2021,13.6,15.2,12.4,19.0,6.6,4.4,33.0,,34.5,70.2,16.7 +New York City,2021,12.0,12.9,9.1,20.2,6.3,4.1,28.8,,25.7,59.5,15.6 +REGION: Capital Region,2021,9.8,11.8,10.5,26.4,7.7,7.2,30.1,,34.0,65.8,18.7 +Statewide,2021,11.4,11.6,9.8,23.6,7.4,5.5,30.5,,29.1,63.6,17.0 diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/validation_config.json b/statvar_imports/us_newyork/ny_brfss_health_indicators/validation_config.json new file mode 100644 index 0000000000..39a6784647 --- /dev/null +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/validation_config.json @@ -0,0 +1,13 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Strictly enforce historical deletion average threshold of 0.1%", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.1 + } + } + ] +} \ No newline at end of file From 2bc656814a7c40f3f76c2b48eba2ac048a82805e Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Wed, 9 Sep 2026 08:49:52 +0000 Subject: [PATCH 02/11] gemini comments resolved --- .../ny_brfss_health_indicators/download.py | 42 ++++--------------- .../test_data/sample_expected_output.csv | 35 +++++----------- .../test_data/sample_expected_output.tmcf | 2 +- .../sample_expected_output_stat_vars.mcf | 24 +---------- ...ample_expected_output_stat_vars_schema.mcf | 7 +--- .../test_data/sample_input.csv | 28 ++++++++++--- 6 files changed, 43 insertions(+), 95 deletions(-) diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py b/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py index cd7009676b..d86b6a276c 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py @@ -20,7 +20,6 @@ """ import os -import sys from absl import app from absl import flags from absl import logging @@ -29,6 +28,7 @@ from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry + FLAGS = flags.FLAGS flags.DEFINE_string( @@ -42,21 +42,6 @@ 'Health Data NY BRFSS Socrata API endpoint.', ) -# Mapping of NY DOH short indicator names to PV map column names -INDICATOR_COLUMN_MAP = { - 'Diabetes': 'Percentage of adults with diabetes', - 'Prediabetes': 'Percentage of adults with prediabetes', - 'Asthma': 'Percentage of adults with asthma', - 'Arthritis': 'Percentage of adults with arthritis', - 'Cardiovascular Disease': 'Percentage of adults with cardiovascular disease', - 'COPD': 'Percentage of adults with COPD', - 'High Blood Pressure': 'Percentage of adults with high blood pressure', - 'Elevated Cholesterol': 'Percentage of adults with elevated cholesterol', - 'Obesity': 'Percentage of adults with obesity', - 'Overweight or Obese': 'Percentage of adults overweight or obese', - 'Depressive Disorder': 'Percentage of adults with depression', -} - def create_session() -> requests.Session: """Creates a requests session configured with retries and connection pooling.""" @@ -77,21 +62,6 @@ def create_session() -> requests.Session: return session -def normalize_year(raw_year: str) -> str: - """Normalizes survey year interval to a 4-digit numeric observation year.""" - raw_year = str(raw_year).strip() - if '-' in raw_year: - parts = raw_year.split('-') - start = parts[0].strip() - end = parts[1].strip() - if len(end) == 2 and len(start) == 4: - return f'{start[:2]}{end}' - if len(end) == 4: - return end - return end - return raw_year - - def atomic_to_csv(df: pd.DataFrame, target_path: str) -> None: """Writes a DataFrame to a CSV file atomically via a temporary file.""" temp_path = f'{target_path}.tmp' @@ -116,10 +86,12 @@ def download_health_indicators(endpoint: str, output_dir: str) -> tuple[int, lis logging.info('GET %s with params %s', endpoint, params) resp = session.get(endpoint, params=params, timeout=60) if resp.status_code != 200: - logging.fatal( + logging.error( 'Health Data NY API returned HTTP %d: %s', resp.status_code, resp.text[:200] ) - return 0, [] + raise RuntimeError( + f'Health Data NY API returned HTTP {resp.status_code}: {resp.text[:200]}' + ) chunk = resp.json() if not chunk: @@ -131,8 +103,8 @@ def download_health_indicators(endpoint: str, output_dir: str) -> tuple[int, lis offset += limit if not records: - logging.fatal('Health Data NY API returned 0 records.') - return 0, [] + logging.error('Health Data NY API returned 0 records.') + raise RuntimeError('Health Data NY API returned 0 records.') logging.info('Received %d total raw records from Health Data NY API.', len(records)) raw_df = pd.DataFrame(records) diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output.csv b/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output.csv index e71b82b96c..a1f1d7d3ef 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output.csv +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output.csv @@ -1,31 +1,18 @@ observationAbout,observationDate,value,variableMeasured -geoId/36001,2021,9.2,dcid:Percent_Person_WithDiabetes geoId/36001,2021,11.7,dcid:Percent_Person_WithPrediabetes -geoId/36001,2021,11.3,dcid:Percent_Person_WithAsthma -geoId/36001,2021,23.7,dcid:Percent_Person_WithArthritis +geoId/36001,2021,9.2,dcid:Percent_Person_WithDiabetes geoId/36001,2021,7.4,dcid:Percent_Person_WithCardiovascularDisease -geoId/36001,2021,6.7,dcid:Percent_Person_WithChronicObstructivePulmonaryDisease -geoId/36001,2021,29.6,dcid:Percent_Person_WithHighBloodPressure -geoId/36001,2021,30.1,dcid:Percent_Person_WithObesity -geoId/36001,2021,61.9,dcid:Percent_Person_WithOverweightOrObese -geoId/36001,2021,21.2,dcid:Percent_Person_Years18Onwards_WithDepression -geoId/36005,2021,13.6,dcid:Percent_Person_WithDiabetes +geoId/36001,2021,21.2,dcid:Percent_Person_18OrMoreYears_WithDepression +geoId/36001,2021,30.1,dcid:Percent_Person_Obesity +geoId/36001,2021,11.3,dcid:Percent_Person_WithAsthma geoId/36005,2021,15.2,dcid:Percent_Person_WithPrediabetes geoId/36005,2021,12.4,dcid:Percent_Person_WithAsthma -geoId/36005,2021,19,dcid:Percent_Person_WithArthritis +geoId/36005,2021,13.6,dcid:Percent_Person_WithDiabetes geoId/36005,2021,6.6,dcid:Percent_Person_WithCardiovascularDisease -geoId/36005,2021,4.4,dcid:Percent_Person_WithChronicObstructivePulmonaryDisease -geoId/36005,2021,33,dcid:Percent_Person_WithHighBloodPressure -geoId/36005,2021,34.5,dcid:Percent_Person_WithObesity -geoId/36005,2021,70.2,dcid:Percent_Person_WithOverweightOrObese -geoId/36005,2021,16.7,dcid:Percent_Person_Years18Onwards_WithDepression -geoId/3651000,2021,12,dcid:Percent_Person_WithDiabetes -geoId/3651000,2021,12.9,dcid:Percent_Person_WithPrediabetes +geoId/36005,2021,16.7,dcid:Percent_Person_18OrMoreYears_WithDepression +geoId/36005,2021,34.5,dcid:Percent_Person_Obesity geoId/3651000,2021,9.1,dcid:Percent_Person_WithAsthma -geoId/3651000,2021,20.2,dcid:Percent_Person_WithArthritis -geoId/3651000,2021,6.3,dcid:Percent_Person_WithCardiovascularDisease -geoId/3651000,2021,4.1,dcid:Percent_Person_WithChronicObstructivePulmonaryDisease -geoId/3651000,2021,28.8,dcid:Percent_Person_WithHighBloodPressure -geoId/3651000,2021,25.7,dcid:Percent_Person_WithObesity -geoId/3651000,2021,59.5,dcid:Percent_Person_WithOverweightOrObese -geoId/3651000,2021,15.6,dcid:Percent_Person_Years18Onwards_WithDepression +geoId/3651000,2021,12.9,dcid:Percent_Person_WithPrediabetes +geoId/3651000,2021,12,dcid:Percent_Person_WithDiabetes +geoId/3651000,2021,25.7,dcid:Percent_Person_Obesity +geoId/36001,2014,9.4,dcid:Percent_Person_WithDiabetes diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output.tmcf b/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output.tmcf index 28fa13adbb..b885d89ede 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output.tmcf +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output.tmcf @@ -3,5 +3,5 @@ observationAbout: C:sample_expected_output->observationAbout observationDate: C:sample_expected_output->observationDate value: C:sample_expected_output->value variableMeasured: C:sample_expected_output->variableMeasured -typeOf: dcs:StatVarObservation unit: Percent +typeOf: dcs:StatVarObservation diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output_stat_vars.mcf b/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output_stat_vars.mcf index e6a1b1c452..ba97cb6730 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output_stat_vars.mcf +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output_stat_vars.mcf @@ -1,4 +1,4 @@ -# Auto generated using command: "tools/statvar_importer/stat_var_processor.py --input_data=statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_input.csv --pv_map=statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_pv_map.csv --config_file=statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_metadata.csv --output_path=statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output" on 2026-09-03 14:13:29.017132 +# Auto generated using command: "../../../tools/statvar_importer/stat_var_processor.py --input_data=test_data/sample_input.csv --pv_map=ny_brfss_health_indicators_pv_map.csv --config_file=ny_brfss_health_indicators_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=test_data/sample_expected_output" on 2026-09-09 08:44:05.973952 Node: dcid:Percent_Person_WithCardiovascularDisease typeOf: dcid:StatisticalVariable @@ -7,20 +7,6 @@ measuredProperty: dcid:percent statType: dcid:measuredValue healthOutcome: dcid:CardiovascularDisease -Node: dcid:Percent_Person_WithObesity -typeOf: dcid:StatisticalVariable -populationType: dcid:Person -measuredProperty: dcid:percent -statType: dcid:measuredValue -healthOutcome: dcid:Obesity - -Node: dcid:Percent_Person_WithOverweightOrObese -typeOf: dcid:StatisticalVariable -populationType: dcid:Person -measuredProperty: dcid:percent -statType: dcid:measuredValue -healthOutcome: dcid:OverweightOrObese - Node: dcid:Percent_Person_WithPrediabetes typeOf: dcid:StatisticalVariable populationType: dcid:Person @@ -28,11 +14,3 @@ measuredProperty: dcid:percent statType: dcid:measuredValue healthOutcome: dcid:Prediabetes -Node: dcid:Percent_Person_Years18Onwards_WithDepression -typeOf: dcid:StatisticalVariable -populationType: dcid:Person -measuredProperty: dcid:percent -statType: dcid:measuredValue -age: dcid:Years18Onwards -healthOutcome: dcid:Depression - diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output_stat_vars_schema.mcf b/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output_stat_vars_schema.mcf index 33e0428767..cd434872ce 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output_stat_vars_schema.mcf +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output_stat_vars_schema.mcf @@ -6,10 +6,5 @@ isProvisional: dcs:True Node: dcid:healthOutcome typeOf: dcs:Property name: "healthOutcome" -rangeIncludes: dcid:MedicalConditionIncidentTypeEnum,dcid:BodyMassIndexClassificationEnum - -Node: dcid:OverweightOrObese -typeOf: dcid:HealthOutcomeEnum -name: "Overweight Or Obese" -isProvisional: dcs:True +rangeIncludes: dcid:MedicalConditionIncidentTypeEnum diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_input.csv b/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_input.csv index 0ad2020c07..fa31755460 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_input.csv +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_input.csv @@ -1,6 +1,22 @@ -Location,Year,Percentage of adults with diabetes,Percentage of adults with prediabetes,Percentage of adults with asthma,Percentage of adults with arthritis,Percentage of adults with cardiovascular disease,Percentage of adults with COPD,Percentage of adults with high blood pressure,Percentage of adults with elevated cholesterol,Percentage of adults with obesity,Percentage of adults overweight or obese,Percentage of adults with depression -Albany,2021,9.2,11.7,11.3,23.7,7.4,6.7,29.6,,30.1,61.9,21.2 -Bronx,2021,13.6,15.2,12.4,19.0,6.6,4.4,33.0,,34.5,70.2,16.7 -New York City,2021,12.0,12.9,9.1,20.2,6.3,4.1,28.8,,25.7,59.5,15.6 -REGION: Capital Region,2021,9.8,11.8,10.5,26.4,7.7,7.2,30.1,,34.0,65.8,18.7 -Statewide,2021,11.4,11.6,9.8,23.6,7.4,5.5,30.5,,29.1,63.6,17.0 +years,health_indicator_short_name,geography,region_county,sample_size_n,est_count,est_denominator,unadjusted_rate,cl_95_lower_unadjusted,cl_95_upper_unadjusted,adjusted_rate,cl_95_lower_adjusted,cl_95_upper_adjusted,health_domain,health_indicator,prevention_agenda_pa,ship,dsrip,chirs,fips,health_indicator_sort_key,dsrip_region_sort_key,county_sort_key,location,:@computed_region_43an_4dx5,:@computed_region_9yqb_tdyd,:@computed_region_assa_msit,:@computed_region_5edz_4hdv,chirs_rate_type,:@computed_region_8ire_itmf,pa_rate_type,notes,upstate_downstate_sort_key,ship_rate_type,dsrip_rate_type +2021,Prediabetes,County,Albany,681.0,18171.0,155370.0,11.7,8.4,15.0,11.9,8.6,15.2,Prevent Chronic Diseases,Percentage of adults with health care provider diagnosed prediabetes,no,Not Applicable,Not Applicable,no,1.0,59,104.0,1.0,"{'latitude': '42.678066', 'longitude': '-73.814233', 'human_address': '{""address"": """", ""city"": """", ""state"": """", ""zip"": """"}'}",40.0,2030.0,34.0,573.0,,1.0,,,,, +2021,Diabetes,County,Albany,1174.0,23847.0,258799.0,9.2,7.3,11.1,8.7,6.9,10.5,Prevent Chronic Diseases,Percentage of adults with health care provider diagnosed diabetes,no,Not Applicable,Not Applicable,yes,1.0,35,104.0,1.0,"{'latitude': '42.678066', 'longitude': '-73.814233', 'human_address': '{""address"": """", ""city"": """", ""state"": """", ""zip"": """"}'}",40.0,2030.0,34.0,573.0,age-adjusted,1.0,,,,, +2021,Cardiovascular Disease,County,Albany,1175.0,19067.0,258878.0,7.4,5.5,9.3,6.3,4.7,8.0,Prevent Chronic Diseases,"Percentage of adults with cardiovascular disease (heart attack, coronary heart disease, or stroke)",no,Not Applicable,Not Applicable,yes,1.0,65,104.0,1.0,"{'latitude': '42.678066', 'longitude': '-73.814233', 'human_address': '{""address"": """", ""city"": """", ""state"": """", ""zip"": """"}'}",40.0,2030.0,34.0,573.0,age-adjusted,1.0,,,,, +2021,Depressive Disorder,County,Albany,1160.0,53767.0,253778.0,21.2,17.9,24.5,21.1,17.8,24.3,Promote Mental Health and Prevention Substance Abuse,Pecentage of adults reporting a depressive disorder,no,Not Applicable,Not Applicable,no,1.0,100,104.0,1.0,"{'latitude': '42.678066', 'longitude': '-73.814233', 'human_address': '{""address"": """", ""city"": """", ""state"": """", ""zip"": """"}'}",40.0,2030.0,34.0,573.0,,1.0,,,,, +2021,Obesity,County,Albany,974.0,64268.0,213476.0,30.1,26.0,34.2,32.4,28.5,36.4,Prevent Chronic Diseases,Percentage of adults who are obese,yes,Not Applicable,Not Applicable,yes,1.0,21,104.0,1.0,"{'latitude': '42.678066', 'longitude': '-73.814233', 'human_address': '{""address"": """", ""city"": """", ""state"": """", ""zip"": """"}'}",40.0,2030.0,34.0,573.0,age-adjusted,1.0,crude,,,, +2021,Asthma,County,Albany,1161.0,28964.0,256532.0,11.3,8.6,13.9,12.0,9.1,14.8,Prevent Chronic Diseases,Percentage of adults with current asthma,no,Not Applicable,Not Applicable,yes,1.0,68,104.0,1.0,"{'latitude': '42.678066', 'longitude': '-73.814233', 'human_address': '{""address"": """", ""city"": """", ""state"": """", ""zip"": """"}'}",40.0,2030.0,34.0,573.0,age-adjusted,1.0,,,,, +2021,Prediabetes,County,Bronx,618.0,89452.0,586929.0,15.2,12.0,18.5,15.6,12.3,18.8,Prevent Chronic Diseases,Percentage of adults with health care provider diagnosed prediabetes,no,Not Applicable,Not Applicable,no,5.0,59,102.0,3.0,"{'latitude': '40.8448', 'longitude': '-73.8648', 'human_address': '{""address"": """", ""city"": """", ""state"": """", ""zip"": """"}'}",675.0,2032.0,51.0,176.0,,37.0,,,,, +2021,Asthma,County,Bronx,1180.0,135984.0,1096419.0,12.4,10.2,14.6,12.5,10.2,14.7,Prevent Chronic Diseases,Percentage of adults with current asthma,no,Not Applicable,Not Applicable,yes,5.0,68,102.0,3.0,"{'latitude': '40.8448', 'longitude': '-73.8648', 'human_address': '{""address"": """", ""city"": """", ""state"": """", ""zip"": """"}'}",675.0,2032.0,51.0,176.0,age-adjusted,37.0,,,,, +2021,Diabetes,County,Bronx,1183.0,149452.0,1099809.0,13.6,11.4,15.7,13.4,11.3,15.5,Prevent Chronic Diseases,Percentage of adults with health care provider diagnosed diabetes,no,Not Applicable,Not Applicable,yes,5.0,35,102.0,3.0,"{'latitude': '40.8448', 'longitude': '-73.8648', 'human_address': '{""address"": """", ""city"": """", ""state"": """", ""zip"": """"}'}",675.0,2032.0,51.0,176.0,age-adjusted,37.0,,,,, +2021,Cardiovascular Disease,County,Bronx,1186.0,73001.0,1102413.0,6.6,5.1,8.2,6.6,5.0,8.1,Prevent Chronic Diseases,"Percentage of adults with cardiovascular disease (heart attack, coronary heart disease, or stroke)",no,Not Applicable,Not Applicable,yes,5.0,65,102.0,3.0,"{'latitude': '40.8448', 'longitude': '-73.8648', 'human_address': '{""address"": """", ""city"": """", ""state"": """", ""zip"": """"}'}",675.0,2032.0,51.0,176.0,age-adjusted,37.0,,,,, +2021,Depressive Disorder,County,Bronx,1174.0,181734.0,1090697.0,16.7,14.2,19.1,16.9,14.5,19.4,Promote Mental Health and Prevention Substance Abuse,Pecentage of adults reporting a depressive disorder,no,Not Applicable,Not Applicable,no,5.0,100,102.0,3.0,"{'latitude': '40.8448', 'longitude': '-73.8648', 'human_address': '{""address"": """", ""city"": """", ""state"": """", ""zip"": """"}'}",675.0,2032.0,51.0,176.0,,37.0,,,,, +2021,Obesity,County,Bronx,950.0,302090.0,874367.0,34.5,31.1,38.0,34.7,31.2,38.2,Prevent Chronic Diseases,Percentage of adults who are obese,yes,Not Applicable,Not Applicable,yes,5.0,21,102.0,3.0,"{'latitude': '40.8448', 'longitude': '-73.8648', 'human_address': '{""address"": """", ""city"": """", ""state"": """", ""zip"": """"}'}",675.0,2032.0,51.0,176.0,age-adjusted,37.0,crude,,,, +2021,Asthma,New York City,New York City,6781.0,610365.0,6742877.0,9.1,8.2,9.9,9.2,8.3,10.1,Prevent Chronic Diseases,Percentage of adults with current asthma,no,Not Applicable,Not Applicable,yes,,68,,,,,,,,age-adjusted,,,,1.0,, +2021,Prediabetes,New York City,New York City,3665.0,642815.0,4980465.0,12.9,11.6,14.3,14.1,12.7,15.5,Prevent Chronic Diseases,Percentage of adults with health care provider diagnosed prediabetes,no,Not Applicable,Not Applicable,no,,59,,,,,,,,,,,,1.0,, +2021,Diabetes,New York City,New York City,6822.0,817951.0,6795640.0,12.0,11.0,13.1,11.4,10.5,12.4,Prevent Chronic Diseases,Percentage of adults with health care provider diagnosed diabetes,no,Not Applicable,Not Applicable,yes,,35,,,,,,,,age-adjusted,,,,1.0,, +2021,Obesity,New York City,New York City,5662.0,1433036.0,5579249.0,25.7,24.2,27.1,25.6,24.1,27.1,Prevent Chronic Diseases,Percentage of adults who are obese,yes,Not Applicable,Not Applicable,yes,,21,,,,,,,,age-adjusted,,crude,,1.0,, +2013-14,Diabetes,County,Albany,1019.0,23281.0,246849.0,9.4,7.0,12.7,8.8,6.6,11.6,Prevent Chronic Diseases,Percentage of adults with physician diagnosed diabetes,no,no,no,no,1.0,35,104.0,1.0,"{'latitude': '42.678066', 'longitude': '-73.814233', 'human_address': '{""address"": """", ""city"": """", ""state"": """", ""zip"": """"}'}",40.0,2030.0,34.0,573.0,,1.0,,,,, +2021,Diabetes,DSRIP region,REGION: Capital Region,3432.0,78896.0,806884.0,9.8,8.0,11.6,8.6,6.9,10.3,Prevent Chronic Diseases,Percentage of adults with health care provider diagnosed diabetes,no,Not Applicable,Not Applicable,yes,,35,104.0,,,,,,,age-adjusted,,,,,, +2021,Diabetes,Statewide,Statewide,38978.0,1827228.0,15988212.0,11.4,10.9,12.0,10.2,9.7,10.7,Prevent Chronic Diseases,Percentage of adults with health care provider diagnosed diabetes,no,Not Applicable,Not Applicable,yes,,35,,,,,,,,age-adjusted,,,,,, +2024,Provider Discussed Healthy Pregnancy,County,Wyoming,,,,,,,,,,"Promote Healthy Women, Infants, and Children",Percentage of women aged 18-44 years whose health care provider discussed planning a healthy pregnancy,no,Not Applicable,Not Applicable,no,121.0,96,111.0,61.0,"{'latitude': '42.723641', 'longitude': '-78.204473', 'human_address': '{""address"": """", ""city"": """", ""state"": """", ""zip"": """"}'}",415.0,2186.0,59.0,622.0,,,,rates suppressed due to small sample size,,, +2024,Provider Discussed Healthy Pregnancy,County,Washington,,,,,,,,,,"Promote Healthy Women, Infants, and Children",Percentage of women aged 18-44 years whose health care provider discussed planning a healthy pregnancy,no,Not Applicable,Not Applicable,no,115.0,96,106.0,58.0,"{'latitude': '43.289513', 'longitude': '-73.454761', 'human_address': '{""address"": """", ""city"": """", ""state"": """", ""zip"": """"}'}",787.0,2184.0,35.0,857.0,,,,rates suppressed due to small sample size,,, From 3ea521f29a2fe3ea2f77e60e6e1ee395b8c925e7 Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Wed, 9 Sep 2026 09:17:27 +0000 Subject: [PATCH 03/11] CRA fix1 --- .../ny_brfss_health_indicators/README.md | 21 ++- .../ny_brfss_health_indicators/download.py | 156 +++++++++--------- .../ny_brfss_health_indicators_pv_map.csv | 10 ++ 3 files changed, 104 insertions(+), 83 deletions(-) diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md b/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md index 749489a46e..d8926b83fe 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md @@ -44,9 +44,13 @@ statvar_imports/us_newyork/ny_brfss_health_indicators/ ├── validation_config.json # Import validation rules (historical deletion threshold) ├── ny_brfss_health_indicators_metadata.csv # Processor configuration metadata ├── ny_brfss_health_indicators_pv_map.csv # Property-value mapping for all 75 StatVars & 62 counties -├── schema.mcf # Combined local test schema (provisional StatVars & enums) ├── download.py # Multi-year Socrata downloader -├── download_test.py # Downloader unit test suite +├── test_data/ # Sample fixtures for integration testing +│ ├── sample_input.csv # Representative raw slice of API data (21 rows) +│ ├── sample_expected_output.csv # Expected golden observations +│ ├── sample_expected_output.tmcf # Expected golden TMCF +│ ├── sample_expected_output_stat_vars.mcf # Expected generated StatVars +│ └── sample_expected_output_stat_vars_schema.mcf # Expected generated schema ├── input_files/ # Multi-year source datasets from NYSDOH API │ └── ny_brfss_health_indicators_raw.csv # Complete raw unpivoted dataset (all 17,700 records) ├── output_files/ # Generated Data Commons artifacts @@ -81,9 +85,15 @@ python3 ../../../tools/statvar_importer/stat_var_processor.py \ --output_counters=counters/ny_brfss_health_indicators_counters.csv ``` -### Step 3: Run Unit Tests +### Step 3: Run Sample Integration Test +Verify property-value mapping against the sample test fixtures: ```bash -python3 -m unittest download_test.py +python3 ../../../tools/statvar_importer/stat_var_processor.py \ + --input_data="test_data/sample_input.csv" \ + --pv_map=ny_brfss_health_indicators_pv_map.csv \ + --config_file=ny_brfss_health_indicators_metadata.csv \ + --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --output_path=test_data/sample_expected_output ``` ### Step 4: Validate with Data Commons Import Tool @@ -91,7 +101,8 @@ python3 -m unittest download_test.py java -jar ~/Downloads/import_tools_import-tool.jar lint \ output_files/ny_brfss_health_indicators_output.csv \ output_files/ny_brfss_health_indicators_output.tmcf \ - schema.mcf + output_files/ny_brfss_health_indicators_output_stat_vars.mcf \ + output_files/ny_brfss_health_indicators_output_stat_vars_schema.mcf ``` --- diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py b/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py index d86b6a276c..bd2dfdfb0d 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py @@ -28,7 +28,6 @@ from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry - FLAGS = flags.FLAGS flags.DEFINE_string( @@ -44,89 +43,90 @@ def create_session() -> requests.Session: - """Creates a requests session configured with retries and connection pooling.""" - session = requests.Session() - retries = Retry( - total=5, - backoff_factor=1.5, - status_forcelist=[429, 500, 502, 503, 504], - raise_on_status=False, - ) - adapter = HTTPAdapter(max_retries=retries) - session.mount('https://', adapter) - session.mount('http://', adapter) - session.headers.update({ - 'User-Agent': 'Mozilla/5.0 (DataCommons Ingestion; +https://datacommons.org)', - 'Accept': 'application/json, text/csv, */*', - }) - return session + """Creates a requests session configured with retries and connection pooling.""" + session = requests.Session() + retries = Retry( + total=5, + backoff_factor=1.5, + status_forcelist=[429, 500, 502, 503, 504], + raise_on_status=False, + ) + adapter = HTTPAdapter(max_retries=retries) + session.mount('https://', adapter) + session.mount('http://', adapter) + session.headers.update({ + 'User-Agent': + 'Mozilla/5.0 (DataCommons Ingestion; +https://datacommons.org)', + 'Accept': + 'application/json, text/csv, */*', + }) + return session def atomic_to_csv(df: pd.DataFrame, target_path: str) -> None: - """Writes a DataFrame to a CSV file atomically via a temporary file.""" - temp_path = f'{target_path}.tmp' - df.to_csv(temp_path, index=False) - os.replace(temp_path, target_path) - - -def download_health_indicators(endpoint: str, output_dir: str) -> tuple[int, list[str]]: - """Downloads all health indicators for all NY counties and regions.""" - session = create_session() - os.makedirs(output_dir, exist_ok=True) - - records = [] - offset = 0 - limit = 50000 - - while True: - params = { - '$limit': limit, - '$offset': offset, - } - logging.info('GET %s with params %s', endpoint, params) - resp = session.get(endpoint, params=params, timeout=60) - if resp.status_code != 200: - logging.error( - 'Health Data NY API returned HTTP %d: %s', resp.status_code, resp.text[:200] - ) - raise RuntimeError( - f'Health Data NY API returned HTTP {resp.status_code}: {resp.text[:200]}' - ) - - chunk = resp.json() - if not chunk: - break - records.extend(chunk) - logging.info('Fetched %d records (offset %d).', len(chunk), offset) - if len(chunk) < limit: - break - offset += limit - - if not records: - logging.error('Health Data NY API returned 0 records.') - raise RuntimeError('Health Data NY API returned 0 records.') - - logging.info('Received %d total raw records from Health Data NY API.', len(records)) - raw_df = pd.DataFrame(records) - - # Save complete raw unpivoted dataset (all 17,000+ API records) - raw_file = os.path.join(output_dir, 'ny_brfss_health_indicators_raw.csv') - atomic_to_csv(raw_df, raw_file) - saved_files = [raw_file] - logging.info('Wrote raw unpivoted dataset (%d records) -> %s', len(raw_df), raw_file) - - return len(raw_df), saved_files + """Writes a DataFrame to a CSV file atomically via a temporary file.""" + temp_path = f'{target_path}.tmp' + df.to_csv(temp_path, index=False) + os.replace(temp_path, target_path) + + +def download_health_indicators(endpoint: str, + output_dir: str) -> tuple[int, list[str]]: + """Downloads all health indicators for all NY counties and regions.""" + session = create_session() + os.makedirs(output_dir, exist_ok=True) + + records = [] + offset = 0 + limit = 50000 + + while True: + params = { + '$limit': limit, + '$offset': offset, + } + logging.info('GET %s with params %s', endpoint, params) + resp = session.get(endpoint, params=params, timeout=60) + if resp.status_code != 200: + logging.error('Health Data NY API returned HTTP %d: %s', + resp.status_code, resp.text[:200]) + raise RuntimeError( + f'Health Data NY API returned HTTP {resp.status_code}: {resp.text[:200]}' + ) + + chunk = resp.json() + if not chunk: + break + records.extend(chunk) + logging.info('Fetched %d records (offset %d).', len(chunk), offset) + if len(chunk) < limit: + break + offset += limit + + if not records: + logging.error('Health Data NY API returned 0 records.') + raise RuntimeError('Health Data NY API returned 0 records.') + + logging.info('Received %d total raw records from Health Data NY API.', + len(records)) + raw_df = pd.DataFrame(records) + + # Save complete raw unpivoted dataset (all 17,000+ API records) + raw_file = os.path.join(output_dir, 'ny_brfss_health_indicators_raw.csv') + atomic_to_csv(raw_df, raw_file) + saved_files = [raw_file] + logging.info('Wrote raw unpivoted dataset (%d records) -> %s', len(raw_df), + raw_file) + + return len(raw_df), saved_files def main(_): - script_dir = os.path.dirname(os.path.abspath(__file__)) - output_dir = ( - FLAGS.output_dir - if os.path.isabs(FLAGS.output_dir) - else os.path.join(script_dir, FLAGS.output_dir) - ) - download_health_indicators(FLAGS.endpoint, output_dir) + script_dir = os.path.dirname(os.path.abspath(__file__)) + output_dir = (FLAGS.output_dir if os.path.isabs(FLAGS.output_dir) else + os.path.join(script_dir, FLAGS.output_dir)) + download_health_indicators(FLAGS.endpoint, output_dir) if __name__ == '__main__': - app.run(main) + app.run(main) diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_pv_map.csv b/statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_pv_map.csv index 40752a4ebe..a8d026e694 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_pv_map.csv +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_pv_map.csv @@ -6,7 +6,17 @@ unadjusted_rate,value,{Number} DSRIP region,#IgnoreRow,True NYS Exclusive of NYC,#IgnoreRow,True Statewide,#IgnoreRow,True +REGION: Capital Region,#IgnoreRow,True +REGION: Central NY,#IgnoreRow,True +REGION: Finger Lakes,#IgnoreRow,True +REGION: Long Island,#IgnoreRow,True +REGION: Mid-Hudson,#IgnoreRow,True +REGION: Mohawk Valley,#IgnoreRow,True REGION: New York City,#IgnoreRow,True +REGION: North Country,#IgnoreRow,True +REGION: Southern Tier,#IgnoreRow,True +REGION: Tug Hill Seaway,#IgnoreRow,True +REGION: Western NY,#IgnoreRow,True Diabetes,populationType,Person,measuredProperty,percent,healthOutcome,Diabetes,unit,Percent Prediabetes,populationType,Person,measuredProperty,percent,healthOutcome,Prediabetes,unit,Percent Asthma,populationType,Person,measuredProperty,percent,healthOutcome,Asthma,unit,Percent From e17b30ae2142e7a532fa197c5fa14059af006010 Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Wed, 9 Sep 2026 12:31:58 +0000 Subject: [PATCH 04/11] pvmap fix --- .../ny_brfss_health_indicators/README.md | 16 ++--- .../ny_brfss_health_indicators/download.py | 1 + .../ny_brfss_health_indicators_pv_map.csv | 62 +++++++++---------- 3 files changed, 40 insertions(+), 39 deletions(-) diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md b/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md index d8926b83fe..860c2a44d0 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md @@ -10,7 +10,7 @@ This import ingests health indicator prevalence rates across **all 62 counties** * **Socrata API Endpoint**: `https://health.data.ny.gov/resource/jsy7-eb4n.json` * **Geographic Coverage**: **All 62 NY Counties** (`geoId/36001` through `geoId/36123`) and **New York City** (`geoId/3651000`). * **Temporal Coverage**: 2014, 2016, 2018, 2021, and 2024 survey releases. -* **Unit & Measurement Method**: Direct survey unadjusted crude prevalence percentages (`Percent`, `scalingFactor: 100`). +* **Unit & Measurement Method**: Direct survey unadjusted crude prevalence percentages (`Percent`). --- @@ -29,9 +29,9 @@ The import supports **all 75 health indicators** in the NYSDOH dataset across 6 | **Total** | **75** | **13,479 Observations** | ### Statistical Variable Resolution -* **Canonical StatVars (12)**: Already established in the Data Commons Knowledge Graph (e.g. `Percent_Person_WithDiabetes`, `Percent_Person_WithAsthma`, `Percent_Person_WithArthritis`, `Percent_Person_WithChronicObstructivePulmonaryDisease`, `Percent_Person_WithHighBloodPressure`, `Percent_Person_WithHighCholesterol`, `Percent_Person_Obesity`, `Percent_Person_18OrMoreYears_WithDepression`, `Percent_Person_Smoking`, `Percent_Person_BingeDrinking`, `Percent_Person_18OrMoreYears_WithPoorGeneralHealth`, `Percent_Person_WithMentalHealthNotGood`, `Percent_Person_WithPhysicalHealthNotGood`). -* **Provisional StatVars (63)**: Synthesized with formal property-value graphs and emitted into `output_files/ny_brfss_health_indicators_output_stat_vars.mcf`. -* **Supporting Schema Nodes (58)**: Provisional enums, properties, and types emitted into `output_files/ny_brfss_health_indicators_output_stat_vars_schema.mcf`. +* **Canonical StatVars (20)**: Reused directly from the Data Commons Knowledge Graph (e.g. `Percent_Person_WithDiabetes`, `Percent_Person_WithAsthma`, `Percent_Person_WithArthritis`, `Percent_Person_WithChronicObstructivePulmonaryDisease`, `Percent_Person_WithHighBloodPressure`, `Percent_Person_WithHighCholesterol`, `Percent_Person_Obesity`, `Percent_Person_18OrMoreYears_WithDepression`, `Percent_Person_Smoking`, `Percent_Person_BingeDrinking`, `Percent_Person_18OrMoreYears_WithPoorGeneralHealth`, `Percent_Person_WithMentalHealthNotGood`, `Percent_Person_WithPhysicalHealthNotGood`, `Percent_Person_18OrMoreYears_WithAnyDisability`, `Percent_Person_ReceivedDentalVisit`, `Percent_Person_ReceivedCholesterolScreening`, `Percent_Person_50To74Years_Female_ReceivedMammography`, `Percent_Person_21To65Years_Female_ReceivedCervicalCancerScreening`, `Percent_Person_50To75Years_ReceivedColorectalCancerScreening`, `Percent_Person_18OrMoreYears_WithHighBloodPressure_ReceivedTakingBloodPressureMedication`). +* **Provisional StatVars (55)**: Synthesized with formal property-value graphs and emitted into `output_files/ny_brfss_health_indicators_output_stat_vars.mcf`. +* **Supporting Schema Nodes (56)**: Provisional enums, properties, and types emitted into `output_files/ny_brfss_health_indicators_output_stat_vars_schema.mcf`. --- @@ -56,8 +56,8 @@ statvar_imports/us_newyork/ny_brfss_health_indicators/ ├── output_files/ # Generated Data Commons artifacts │ ├── ny_brfss_health_indicators_output.csv # Cleaned StatVarObservations (13,479 rows) │ ├── ny_brfss_health_indicators_output.tmcf # Template MCF mapping file -│ ├── ny_brfss_health_indicators_output_stat_vars.mcf # 63 Provisional StatVar nodes -│ └── ny_brfss_health_indicators_output_stat_vars_schema.mcf # 58 Provisional schema definitions +│ ├── ny_brfss_health_indicators_output_stat_vars.mcf # 55 Provisional StatVar nodes +│ └── ny_brfss_health_indicators_output_stat_vars_schema.mcf # 56 Provisional schema definitions └── counters/ # Processor execution statistics └── ny_brfss_health_indicators_counters.csv ``` @@ -111,6 +111,6 @@ java -jar ~/Downloads/import_tools_import-tool.jar lint \ * **Observations Generated**: **13,479** `StatVarObservation` records. * **Geographies Resolved**: **63 unique entities** (all 62 NY counties `geoId/36001` - `geoId/36123` plus New York City `geoId/3651000`). -* **Non-County Regions Dropped**: **2,831 records** (sub-state DSRIP regions, Rest of State, and Statewide rows cleanly dropped by PV map). -* **Statistical Variables Generated**: **75 unique variables** (12 matched to existing canonical DCIDs, 63 provisional StatVars, 58 schema enums/properties). +* **Non-County Regions Dropped**: **3,081 records** (sub-state DSRIP regions, Rest of State, and Statewide rows cleanly dropped by PV map). +* **Statistical Variables Generated**: **75 unique variables** (20 matched to existing canonical DCIDs, 55 provisional StatVars, 56 schema enums/properties). * **Linter Status**: **0 fatal, 0 errors, 0 missing references** (`NumRowSuccesses: 13,479 / 13,479`). diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py b/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py index bd2dfdfb0d..a180cfd6a8 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py @@ -84,6 +84,7 @@ def download_health_indicators(endpoint: str, params = { '$limit': limit, '$offset': offset, + '$order': ':id', } logging.info('GET %s with params %s', endpoint, params) resp = session.get(endpoint, params=params, timeout=60) diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_pv_map.csv b/statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_pv_map.csv index a8d026e694..ddef13a5c7 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_pv_map.csv +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_pv_map.csv @@ -17,14 +17,14 @@ REGION: North Country,#IgnoreRow,True REGION: Southern Tier,#IgnoreRow,True REGION: Tug Hill Seaway,#IgnoreRow,True REGION: Western NY,#IgnoreRow,True -Diabetes,populationType,Person,measuredProperty,percent,healthOutcome,Diabetes,unit,Percent +Diabetes,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthOutcome,Diabetes,unit,Percent Prediabetes,populationType,Person,measuredProperty,percent,healthOutcome,Prediabetes,unit,Percent -Asthma,populationType,Person,measuredProperty,percent,healthOutcome,Asthma,unit,Percent -Arthritis,populationType,Person,measuredProperty,percent,healthOutcome,Arthritis,unit,Percent +Asthma,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthOutcome,Asthma,unit,Percent +Arthritis,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthOutcome,Arthritis,unit,Percent Cardiovascular Disease,populationType,Person,measuredProperty,percent,healthOutcome,CardiovascularDisease,unit,Percent -COPD,populationType,Person,measuredProperty,percent,healthOutcome,ChronicObstructivePulmonaryDisease,unit,Percent -High Blood Pressure,populationType,Person,measuredProperty,percent,healthOutcome,HighBloodPressure,unit,Percent -Elevated Cholesterol,populationType,Person,measuredProperty,percent,healthOutcome,HighCholesterol,unit,Percent +COPD,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthOutcome,ChronicObstructivePulmonaryDisease,unit,Percent +High Blood Pressure,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthOutcome,HighBloodPressure,unit,Percent +Elevated Cholesterol,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthOutcome,HighCholesterol,unit,Percent Obesity,dcid,Percent_Person_Obesity,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthBehavior,Obesity,unit,Percent Overweight or Obese,populationType,Person,measuredProperty,percent,healthOutcome,OverweightOrObese,unit,Percent Depressive Disorder,dcid,Percent_Person_18OrMoreYears_WithDepression,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthOutcome,Depression,unit,Percent @@ -33,29 +33,29 @@ Active Transportation,populationType,Person,measuredProperty,percent,healthBehav Asthma Ever,populationType,Person,measuredProperty,percent,healthOutcome,AsthmaEver,unit,Percent Binge Drinking,dcid,Percent_Person_BingeDrinking,populationType,Person,measuredProperty,percent,healthBehavior,BingeDrinking,unit,Percent Binge or Heavy Drinking,populationType,Person,measuredProperty,percent,healthBehavior,BingeOrHeavyDrinking,unit,Percent -Breast Cancer Screening,dcid,Percent_Person_50To74Years_Female_ReceivedMammography,populationType,Person,gender,Female,age,Years50To74,measuredProperty,percent,healthCareUtilization,Mammography,unit,Percent -Breast Cancer Screening Low Income 50-74,populationType,Person,gender,Female,age,Years50To74,income,USDollar0To24999,measuredProperty,percent,healthCareUtilization,Mammography,unit,Percent -Cardiovascular Disease Disability,populationType,Person,healthStatus,WithDisability,measuredProperty,percent,healthOutcome,CardiovascularDisease,unit,Percent -Cervical Cancer Screening,dcid,Percent_Person_21To65Years_Female_ReceivedCervicalCancerScreening,populationType,Person,gender,Female,age,Years21To65,measuredProperty,percent,healthCareUtilization,CervicalCancerScreening,unit,Percent -Cervical Cancer Screening (2012 guidelines),populationType,Person,gender,Female,age,Years21To65,measuredProperty,percent,healthCareUtilization,CervicalCancerScreening2012Guidelines,unit,Percent -Cervical Cancer Screening (2012 guidelines) 21-29,populationType,Person,gender,Female,age,Years21To29,measuredProperty,percent,healthCareUtilization,CervicalCancerScreening2012Guidelines,unit,Percent -Cervical Cancer Screening (pre-2012 guidelines),populationType,Person,gender,Female,age,Years21To65,measuredProperty,percent,healthCareUtilization,CervicalCancerScreeningPre2012Guidelines,unit,Percent -Cervical Cancer Screening (pre-2012 guidelines) Low Income,populationType,Person,gender,Female,age,Years21To65,income,USDollar0To24999,measuredProperty,percent,healthCareUtilization,CervicalCancerScreeningPre2012Guidelines,unit,Percent -Cholesterol Checked,dcid,Percent_Person_ReceivedCholesterolScreening,populationType,Person,measuredProperty,percent,healthCareUtilization,CholesterolScreening,unit,Percent +Breast Cancer Screening,dcid,Percent_Person_50To74Years_Female_ReceivedMammography,populationType,Person,gender,Female,age,Years50To74,measuredProperty,percent,healthPrevention,Mammography,unit,Percent +Breast Cancer Screening Low Income 50-74,populationType,Person,gender,Female,age,Years50To74,income,USDollar0To24999,measuredProperty,percent,healthPrevention,Mammography,unit,Percent +Cardiovascular Disease Disability,populationType,Person,disabilityStatus,AnyDisability,measuredProperty,percent,healthOutcome,CardiovascularDisease,unit,Percent +Cervical Cancer Screening,dcid,Percent_Person_21To65Years_Female_ReceivedCervicalCancerScreening,populationType,Person,gender,Female,age,Years21To65,measuredProperty,percent,healthPrevention,CervicalCancerScreening,unit,Percent +Cervical Cancer Screening (2012 guidelines),populationType,Person,gender,Female,age,Years21To65,measuredProperty,percent,healthPrevention,CervicalCancerScreening2012Guidelines,unit,Percent +Cervical Cancer Screening (2012 guidelines) 21-29,populationType,Person,gender,Female,age,Years21To29,measuredProperty,percent,healthPrevention,CervicalCancerScreening2012Guidelines,unit,Percent +Cervical Cancer Screening (pre-2012 guidelines),populationType,Person,gender,Female,age,Years21To65,measuredProperty,percent,healthPrevention,CervicalCancerScreeningPre2012Guidelines,unit,Percent +Cervical Cancer Screening (pre-2012 guidelines) Low Income,populationType,Person,gender,Female,age,Years21To65,income,USDollar0To24999,measuredProperty,percent,healthPrevention,CervicalCancerScreeningPre2012Guidelines,unit,Percent +Cholesterol Checked,dcid,Percent_Person_ReceivedCholesterolScreening,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthPrevention,CholesterolScreening,unit,Percent Chronic Disease Self-Management,populationType,Person,measuredProperty,percent,healthBehavior,ChronicDiseaseSelfManagement,unit,Percent -Colorectal Cancer Screening,dcid,Percent_Person_50To75Years_ReceivedColorectalCancerScreening,populationType,Person,age,Years50To75,measuredProperty,percent,healthCareUtilization,ColorectalCancerScreening,unit,Percent -Colorectal Cancer Screening 50-64,populationType,Person,age,Years50To64,measuredProperty,percent,healthCareUtilization,ColorectalCancerScreening,unit,Percent -Colorectal Cancer Screening Low Income,populationType,Person,age,Years50To75,income,USDollar0To24999,measuredProperty,percent,healthCareUtilization,ColorectalCancerScreening,unit,Percent +Colorectal Cancer Screening,dcid,Percent_Person_50To75Years_ReceivedColorectalCancerScreening,populationType,Person,age,Years50To75,measuredProperty,percent,healthPrevention,ColorectalCancerScreening,unit,Percent +Colorectal Cancer Screening 50-64,populationType,Person,age,Years50To64,measuredProperty,percent,healthPrevention,ColorectalCancerScreening,unit,Percent +Colorectal Cancer Screening Low Income,populationType,Person,age,Years50To75,income,USDollar0To24999,measuredProperty,percent,healthPrevention,ColorectalCancerScreening,unit,Percent Current Cannabis Use,populationType,Person,measuredProperty,percent,healthBehavior,CannabisUse,unit,Percent Current Medical Cannabis Use,populationType,Person,measuredProperty,percent,healthBehavior,MedicalCannabisUse,unit,Percent -Current Smoking,dcid,Percent_Person_Smoking,populationType,Person,measuredProperty,percent,healthBehavior,SmokingStatus_CurrentSmoker,unit,Percent -Current Smoking 18-24,populationType,Person,age,Years18To24,measuredProperty,percent,healthBehavior,SmokingStatus_CurrentSmoker,unit,Percent -Current Smoking Disability,populationType,Person,healthStatus,WithDisability,measuredProperty,percent,healthBehavior,SmokingStatus_CurrentSmoker,unit,Percent -Current Smoking Low Income,populationType,Person,income,USDollar0To24999,measuredProperty,percent,healthBehavior,SmokingStatus_CurrentSmoker,unit,Percent -Current Smoking Poor Mental Health,populationType,Person,healthStatus,PoorMentalHealth,measuredProperty,percent,healthBehavior,SmokingStatus_CurrentSmoker,unit,Percent +Current Smoking,dcid,Percent_Person_Smoking,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthBehavior,Smoking,unit,Percent +Current Smoking 18-24,populationType,Person,age,Years18To24,measuredProperty,percent,healthBehavior,Smoking,unit,Percent +Current Smoking Disability,populationType,Person,disabilityStatus,AnyDisability,measuredProperty,percent,healthBehavior,Smoking,unit,Percent +Current Smoking Low Income,populationType,Person,income,USDollar0To24999,measuredProperty,percent,healthBehavior,Smoking,unit,Percent +Current Smoking Poor Mental Health,populationType,Person,healthStatus,PoorMentalHealth,measuredProperty,percent,healthBehavior,Smoking,unit,Percent DWI,populationType,Person,measuredProperty,percent,healthBehavior,DrivingWhileIntoxicated,unit,Percent Diabetes and Prediabetes Testing,populationType,Person,measuredProperty,percent,healthCareUtilization,DiabetesOrPrediabetesTesting,unit,Percent -Disability,dcid,Percent_Person_18OrMoreYears_WithAnyDisability,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthStatus,WithDisability,unit,Percent +Disability,dcid,Percent_Person_18OrMoreYears_WithAnyDisability,populationType,Person,age,Years18Onwards,measuredProperty,percent,disabilityStatus,AnyDisability,unit,Percent Fall within Past Year 65+,populationType,Person,age,Years65Onwards,measuredProperty,percent,healthOutcome,FallPastYear,unit,Percent Falls Resulting in Injury 65+,populationType,Person,age,Years65Onwards,measuredProperty,percent,healthOutcome,FallWithInjuryPastYear,unit,Percent Fast Food Consumption,populationType,Person,measuredProperty,percent,healthBehavior,FastFoodConsumption3OrMoreTimesPerWeek,unit,Percent @@ -68,27 +68,27 @@ HCV Ever Tested 47-68,populationType,Person,age,Years47To68,measuredProperty,per Health Care Coverage,populationType,Person,age,Years18To64,measuredProperty,percent,healthInsurance,WithHealthInsurance,unit,Percent Health Care Coverage Female 18-64,populationType,Person,gender,Female,age,Years18To64,measuredProperty,percent,healthInsurance,WithHealthInsurance,unit,Percent Health Care Provider,populationType,Person,measuredProperty,percent,healthCareProviderStatus,HasPersonalHealthCareProvider,unit,Percent -High Blood Pressure Taking Medication,dcid,Percent_Person_18OrMoreYears_WithHighBloodPressure_ReceivedTakingBloodPressureMedication,populationType,Person,age,Years18Onwards,measuredProperty,percent,medicalCondition,HighBloodPressureMedication,unit,Percent +High Blood Pressure Taking Medication,dcid,Percent_Person_18OrMoreYears_WithHighBloodPressure_ReceivedTakingBloodPressureMedication,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthOutcome,HighBloodPressure,healthPrevention,TakingBloodPressureMedication,unit,Percent Housing Insecurity,populationType,Person,measuredProperty,percent,socialSecurityStatus,HousingInsecurityPast12Months,unit,Percent Housing or Utility Insecurity,populationType,Person,measuredProperty,percent,socialSecurityStatus,HousingOrUtilityInsecurityPast12Months,unit,Percent Leisure-Time Physical Activity,populationType,Person,measuredProperty,percent,healthBehavior,LeisureTimePhysicalActivity,unit,Percent Limitation Status,populationType,Person,measuredProperty,percent,healthStatus,WithLimitation,unit,Percent No Fruits or Vegetables,populationType,Person,measuredProperty,percent,healthBehavior,ConsumesNoFruitsOrVegetables,unit,Percent No Medical Care Due to Cost,populationType,Person,measuredProperty,percent,healthCareAccessStatus,DidNotReceiveCareDueToCost,unit,Percent -Obesity Disability,populationType,Person,healthStatus,WithDisability,measuredProperty,percent,healthBehavior,Obesity,unit,Percent +Obesity Disability,populationType,Person,disabilityStatus,AnyDisability,measuredProperty,percent,healthBehavior,Obesity,unit,Percent Obesity Limitation Status,populationType,Person,healthStatus,WithLimitation,measuredProperty,percent,healthBehavior,Obesity,unit,Percent Obesity Low Income,populationType,Person,income,USDollar0To24999,measuredProperty,percent,healthBehavior,Obesity,unit,Percent PSA Screening,populationType,Person,gender,Male,age,Years55To69,measuredProperty,percent,healthCareUtilization,ProstateSpecificAntigenScreeningSharedDecision,unit,Percent Pneumonia Shot 65+,populationType,Person,age,Years65Onwards,measuredProperty,percent,healthCareUtilization,PneumococcalVaccineEver,unit,Percent Poor General Health,dcid,Percent_Person_18OrMoreYears_WithPoorGeneralHealth,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthOutcome,PoorGeneralHealth,unit,Percent -Poor Mental Health,dcid,Percent_Person_WithMentalHealthNotGood,populationType,Person,measuredProperty,percent,healthOutcome,MentalHealthNotGood,unit,Percent -Poor Physical Health,dcid,Percent_Person_WithPhysicalHealthNotGood,populationType,Person,measuredProperty,percent,healthOutcome,PhysicalHealthNotGood,unit,Percent +Poor Mental Health,dcid,Percent_Person_WithMentalHealthNotGood,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthOutcome,MentalHealthNotGood,unit,Percent +Poor Physical Health,dcid,Percent_Person_WithPhysicalHealthNotGood,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthOutcome,PhysicalHealthNotGood,unit,Percent Provider Discussed Healthy Pregnancy,populationType,Person,gender,Female,age,Years18To44,measuredProperty,percent,healthCareUtilization,ProviderDiscussedHealthyPregnancy,unit,Percent -Recent Checkup,dcid,Percent_Person_ReceivedAnnualCheckup,populationType,Person,age,Years18To64,measuredProperty,percent,healthCareUtilization,RoutineCheckupWithinPastYear,unit,Percent +Recent Checkup,populationType,Person,age,Years18To64,measuredProperty,percent,healthCareUtilization,RoutineCheckupWithinPastYear,unit,Percent Recent Checkup Female 18-44,populationType,Person,gender,Female,age,Years18To44,measuredProperty,percent,healthCareUtilization,RoutineCheckupWithinPastYear,unit,Percent Recent Checkup Female 18-64,populationType,Person,gender,Female,age,Years18To64,measuredProperty,percent,healthCareUtilization,RoutineCheckupWithinPastYear,unit,Percent -Recent Dental Visit,dcid,Percent_Person_ReceivedDentalVisit,populationType,Person,measuredProperty,percent,healthCareUtilization,DentalVisitWithinPastYear,unit,Percent -Recent Dental Visit Female 18-44,populationType,Person,gender,Female,age,Years18To44,measuredProperty,percent,healthCareUtilization,DentalVisitWithinPastYear,unit,Percent +Recent Dental Visit,dcid,Percent_Person_ReceivedDentalVisit,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthPrevention,DentalVisit,unit,Percent +Recent Dental Visit Female 18-44,populationType,Person,gender,Female,age,Years18To44,measuredProperty,percent,healthPrevention,DentalVisit,unit,Percent Sugar-Sweetened Beverages,populationType,Person,measuredProperty,percent,healthBehavior,DailySugarSweetenedBeverages,unit,Percent Walkable Neighborhood,populationType,Person,measuredProperty,percent,environmentStatus,WalkableNeighborhood,unit,Percent e-Cigarettes,populationType,Person,measuredProperty,percent,healthBehavior,ElectronicCigaretteUse,unit,Percent From 354e0b842efe733d7a2c907336798f900dec78b1 Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Thu, 10 Sep 2026 12:20:07 +0000 Subject: [PATCH 05/11] Update README to remove import-tool lint and configure date freshness validations --- .../ny_brfss_health_indicators/README.md | 20 +++++++++---------- .../validation_config.json | 20 ++++++++++++++++++- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md b/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md index 860c2a44d0..e200ac26c4 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md @@ -41,7 +41,7 @@ The import supports **all 75 health indicators** in the NYSDOH dataset across 6 statvar_imports/us_newyork/ny_brfss_health_indicators/ ├── README.md # Comprehensive import documentation ├── manifest.json # Data Commons import automation manifest -├── validation_config.json # Import validation rules (historical deletion threshold) +├── validation_config.json # Import validation rules (historical deletion and date freshness) ├── ny_brfss_health_indicators_metadata.csv # Processor configuration metadata ├── ny_brfss_health_indicators_pv_map.csv # Property-value mapping for all 75 StatVars & 62 counties ├── download.py # Multi-year Socrata downloader @@ -96,21 +96,19 @@ python3 ../../../tools/statvar_importer/stat_var_processor.py \ --output_path=test_data/sample_expected_output ``` -### Step 4: Validate with Data Commons Import Tool -```bash -java -jar ~/Downloads/import_tools_import-tool.jar lint \ - output_files/ny_brfss_health_indicators_output.csv \ - output_files/ny_brfss_health_indicators_output.tmcf \ - output_files/ny_brfss_health_indicators_output_stat_vars.mcf \ - output_files/ny_brfss_health_indicators_output_stat_vars_schema.mcf -``` +### Validation Configuration + +Validation is configured in `validation_config.json`: +* `check_deleted_records_percent`: Strictly enforces a historical deletion average threshold of `0.1%`. Note that on the initial import run, this rule expectedly reports missing differ summary because there is no prior version in prod GCS to diff against; it is configured to safeguard future recurring refreshes. Per consensus on initial imports, golden regression files are omitted from initial submission. +* `check_max_date_freshness`: Enforces date freshness (`CAST(MaxDate AS INTEGER) >= 2024`) across active recurring health indicator StatVars (e.g. Diabetes, Obesity, Smoking, Asthma, Hypertension, Cardiovascular Disease, Depression, Dental Visits, Routine Checkups) to ensure the latest published survey wave is always captured. +* `check_import_max_date_freshness`: Enforces that the overall import contains observations through at least 2024 (`CAST(max_date AS INTEGER) >= 2024`). --- -## 5. Verification Results +## 5. Summary Results * **Observations Generated**: **13,479** `StatVarObservation` records. * **Geographies Resolved**: **63 unique entities** (all 62 NY counties `geoId/36001` - `geoId/36123` plus New York City `geoId/3651000`). * **Non-County Regions Dropped**: **3,081 records** (sub-state DSRIP regions, Rest of State, and Statewide rows cleanly dropped by PV map). * **Statistical Variables Generated**: **75 unique variables** (20 matched to existing canonical DCIDs, 55 provisional StatVars, 56 schema enums/properties). -* **Linter Status**: **0 fatal, 0 errors, 0 missing references** (`NumRowSuccesses: 13,479 / 13,479`). + diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/validation_config.json b/statvar_imports/us_newyork/ny_brfss_health_indicators/validation_config.json index 39a6784647..9c13cbe0a8 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/validation_config.json +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/validation_config.json @@ -8,6 +8,24 @@ "params": { "threshold": 0.1 } + }, + { + "rule_id": "check_max_date_freshness", + "description": "Verify MaxDate is at least 2024 for recurring health indicators", + "validator": "SQL_VALIDATOR", + "params": { + "query": "SELECT StatVar, MaxDate FROM stats WHERE StatVar IN ('Percent_Person_WithDiabetes', 'Percent_Person_Obesity', 'Percent_Person_Smoking', 'Percent_Person_WithAsthma', 'Percent_Person_WithArthritis', 'Percent_Person_WithHighBloodPressure', 'Percent_Person_WithCardiovascularDisease', 'Percent_Person_WithChronicObstructivePulmonaryDisease', 'Percent_Person_18OrMoreYears_WithDepression', 'Percent_Person_18OrMoreYears_WithPoorGeneralHealth', 'Percent_Person_BingeDrinking', 'Percent_Person_LeisureTimePhysicalActivity', 'Percent_Person_ReceivedDentalVisit', 'Percent_Person_WithPrediabetes', 'Percent_Person_18To64Years_WithHealthInsurance', 'Percent_Person_18OrMoreYears_WithAnyDisability', 'Percent_Person_Years18To64_RoutineCheckupWithinPastYear', 'Percent_Person_50To74Years_Female_ReceivedMammography')", + "condition": "CAST(MaxDate AS INTEGER) >= 2024" + } + }, + { + "rule_id": "check_import_max_date_freshness", + "description": "Verify the import contains latest survey wave observations through at least 2024", + "validator": "SQL_VALIDATOR", + "params": { + "query": "SELECT MAX(MaxDate) AS max_date FROM stats", + "condition": "CAST(max_date AS INTEGER) >= 2024" + } } ] -} \ No newline at end of file +} From b408e5ef05be5809224b9316ee8a77a81729dfa9 Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Fri, 11 Sep 2026 07:03:16 +0000 Subject: [PATCH 06/11] crafixes --- .../ny_brfss_health_indicators/README.md | 129 ++++++++---------- .../ny_brfss_health_indicators/manifest.json | 5 +- .../validation_config.json | 1 + 3 files changed, 64 insertions(+), 71 deletions(-) diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md b/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md index e200ac26c4..880eb7effa 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md @@ -1,80 +1,71 @@ -# NewYork_BRFSS_Health_Indicators +# New York BRFSS Health Indicators Data ## 1. Import Overview -This import ingests health indicator prevalence rates across **all 62 counties** in New York State and New York City. The dataset is sourced directly from the **New York State Department of Health (NYSDOH) Behavioral Risk Factor Surveillance System (eBRFSS)**. +This project processes and imports health indicator prevalence rates across all 62 counties in New York State and New York City, provided by the New York State Department of Health. The dataset provides annual county-level estimates across 5 survey releases (2014, 2016, 2018, 2021, and 2024). -* **Import Directory**: `statvar_imports/us_newyork/ny_brfss_health_indicators/` -* **Import Name**: `NewYork_BRFSS_Health_Indicators` -* **Dataset Landing Page**: [https://health.data.ny.gov/Health/Behavioral-Risk-Factor-Surveillance-System-BRFSS-H/jsy7-eb4n/about_data](https://health.data.ny.gov/Health/Behavioral-Risk-Factor-Surveillance-System-BRFSS-H/jsy7-eb4n/about_data) -* **Socrata API Endpoint**: `https://health.data.ny.gov/resource/jsy7-eb4n.json` -* **Geographic Coverage**: **All 62 NY Counties** (`geoId/36001` through `geoId/36123`) and **New York City** (`geoId/3651000`). -* **Temporal Coverage**: 2014, 2016, 2018, 2021, and 2024 survey releases. -* **Unit & Measurement Method**: Direct survey unadjusted crude prevalence percentages (`Percent`). +* **Source URL**: [https://health.data.ny.gov/Health/Behavioral-Risk-Factor-Surveillance-System-BRFSS-H/jsy7-eb4n/about_data](https://health.data.ny.gov/Health/Behavioral-Risk-Factor-Surveillance-System-BRFSS-H/jsy7-eb4n/about_data) +* **Import Type**: Automated +* **Source Data Availability**: Data is available for 2014, 2016, 2018, 2021, and 2024. +* **Release Frequency**: Periodic survey releases (biennial / triennial survey waves). +* **Notes**: This dataset provides county-level estimates across 75 health indicators spanning chronic disease, mental health, substance use, disability, immunizations, and social determinants of health. The data originates from the Behavioral Risk Factor Surveillance System (BRFSS). --- -## 2. Health Indicators & Statistical Variables +## 2. Preprocessing Steps -The import supports **all 75 health indicators** in the NYSDOH dataset across 6 primary health domains: +The import process involves querying the NYSDOH Socrata API and running a processing script on downloaded source data to generate the final artifacts for ingestion. -| Health Domain | Indicators | Key Topics Covered | -|---|:---:|---| -| **Prevent Chronic Diseases** | 42 | Arthritis, Asthma, Cardiovascular Disease, COPD, Diabetes, High Blood Pressure, Obesity, Overweight, Current Smoking (all age/income/disability subsets), e-Cigarettes, Breast/Cervical/Colorectal/PSA Screenings, Physical Activity, Fast Food, Sugary Drinks, Fruits/Vegetables | -| **Improve Health Status & Reduce Disparities** | 13 | Disability (6-question ACA standard), Limitation Status, Health Care Coverage, Personal Health Care Provider, Annual Checkup, Dental Visit, Medical Care Cost Barrier, Poor General & Physical Health, Food Insecurity, Food Security, Housing Insecurity | -| **Promote Mental Health & Substance Abuse** | 9 | Depressive Disorder, Poor Mental Health ($\ge 14$ days), Binge Drinking, Heavy Drinking, DWI, Cannabis Use, Medical Cannabis Use, Smoking with Poor Mental Health, Adverse Childhood Experiences (ACEs) | -| **Prevent HIV/STDs, Vaccine-Preventable Diseases & HAIs** | 5 | Influenza Immunization (Ages 18+, Ages 65+), Pneumococcal Immunization (Ages 65+), Hepatitis C (HCV) Testing (All Adults, Ages 47-68) | -| **Promote Healthy Women, Infants, and Children** | 5 | Female Health Care Coverage (18-64), Female Routine Checkups (18-44, 18-64), Female Dental Visits (18-44), Health Care Provider Discussed Healthy Pregnancy | -| **Promote a Healthy and Safe Environment** | 1 | Walkable Neighborhoods Suitable for Physical Activity | -| **Total** | **75** | **13,479 Observations** | +* **Input files**: + * `input_files/`: This directory contains the raw unpivoted data file (`ny_brfss_health_indicators_raw.csv`) containing all 17,700 records across all survey years (2014, 2016, 2018, 2021, 2024). + * `ny_brfss_health_indicators_metadata.csv`: Configuration file for the data processing script specifying column mappings, header row offset, and provenance URL. + * `ny_brfss_health_indicators_pv_map.csv`: Property-value mapping file used by the processor to map indicators and county locations to Data Commons entities. + * `validation_config.json`: Configuration defining historical deletion and date freshness validation rules. + * `schema.mcf`: Consolidated provisional StatVars (55) and schema nodes (56) for Piper CL ingestion. + * `test_data/`: Sample input data and expected output files for integration testing. -### Statistical Variable Resolution -* **Canonical StatVars (20)**: Reused directly from the Data Commons Knowledge Graph (e.g. `Percent_Person_WithDiabetes`, `Percent_Person_WithAsthma`, `Percent_Person_WithArthritis`, `Percent_Person_WithChronicObstructivePulmonaryDisease`, `Percent_Person_WithHighBloodPressure`, `Percent_Person_WithHighCholesterol`, `Percent_Person_Obesity`, `Percent_Person_18OrMoreYears_WithDepression`, `Percent_Person_Smoking`, `Percent_Person_BingeDrinking`, `Percent_Person_18OrMoreYears_WithPoorGeneralHealth`, `Percent_Person_WithMentalHealthNotGood`, `Percent_Person_WithPhysicalHealthNotGood`, `Percent_Person_18OrMoreYears_WithAnyDisability`, `Percent_Person_ReceivedDentalVisit`, `Percent_Person_ReceivedCholesterolScreening`, `Percent_Person_50To74Years_Female_ReceivedMammography`, `Percent_Person_21To65Years_Female_ReceivedCervicalCancerScreening`, `Percent_Person_50To75Years_ReceivedColorectalCancerScreening`, `Percent_Person_18OrMoreYears_WithHighBloodPressure_ReceivedTakingBloodPressureMedication`). -* **Provisional StatVars (55)**: Synthesized with formal property-value graphs and emitted into `output_files/ny_brfss_health_indicators_output_stat_vars.mcf`. -* **Supporting Schema Nodes (56)**: Provisional enums, properties, and types emitted into `output_files/ny_brfss_health_indicators_output_stat_vars_schema.mcf`. +* **Transformation pipeline**: + 1. The raw data is queried from the NYSDOH Socrata API using deterministic pagination (`$order: ':id'`) via `download.py` and placed in the `input_files/` directory. + 2. The `stat_var_processor.py` tool is run on the raw data against `ny_brfss_health_indicators_pv_map.csv` and `ny_brfss_health_indicators_metadata.csv`, referencing canonical schema `gs://unresolved_mcf/scripts/statvar/stat_vars.mcf`. + 3. The processor filters non-county regional rows, resolves county FIPS DCIDs, maps indicators to canonical or provisional StatVars, and generates the final `ny_brfss_health_indicators_output.csv`, `ny_brfss_health_indicators_output.tmcf`, and supporting StatVar and schema MCF files in the `output_files/` directory. + 4. Processor statistics and metrics are recorded in `counters/ny_brfss_health_indicators_counters.csv`. + +* **Data Quality Checks**: + * The `dc_generated/` directory contains `report.json` and `summary_report.csv`, which provide validation and summary statistics for the generated data. + * Automated validation via `validator.py` evaluates the output against `validation_config.json` enforcing the historical deletion threshold (<= 0.1%) and date freshness (`CAST(MaxDate AS INTEGER) >= 2024`). --- -## 3. Directory Layout +## 3. Automated Import -``` -statvar_imports/us_newyork/ny_brfss_health_indicators/ -├── README.md # Comprehensive import documentation -├── manifest.json # Data Commons import automation manifest -├── validation_config.json # Import validation rules (historical deletion and date freshness) -├── ny_brfss_health_indicators_metadata.csv # Processor configuration metadata -├── ny_brfss_health_indicators_pv_map.csv # Property-value mapping for all 75 StatVars & 62 counties -├── download.py # Multi-year Socrata downloader -├── test_data/ # Sample fixtures for integration testing -│ ├── sample_input.csv # Representative raw slice of API data (21 rows) -│ ├── sample_expected_output.csv # Expected golden observations -│ ├── sample_expected_output.tmcf # Expected golden TMCF -│ ├── sample_expected_output_stat_vars.mcf # Expected generated StatVars -│ └── sample_expected_output_stat_vars_schema.mcf # Expected generated schema -├── input_files/ # Multi-year source datasets from NYSDOH API -│ └── ny_brfss_health_indicators_raw.csv # Complete raw unpivoted dataset (all 17,700 records) -├── output_files/ # Generated Data Commons artifacts -│ ├── ny_brfss_health_indicators_output.csv # Cleaned StatVarObservations (13,479 rows) -│ ├── ny_brfss_health_indicators_output.tmcf # Template MCF mapping file -│ ├── ny_brfss_health_indicators_output_stat_vars.mcf # 55 Provisional StatVar nodes -│ └── ny_brfss_health_indicators_output_stat_vars_schema.mcf # 56 Provisional schema definitions -└── counters/ # Processor execution statistics - └── ny_brfss_health_indicators_counters.csv -``` +This import is designed to be fully automated and autorefreshed. Future survey releases are automatically queried, processed, and validated. + +### Automated Steps +1. The automated job triggers annually based on cron schedule `0 0 1 8 *` configured in `manifest.json`. +2. `download.py` queries the NYSDOH Socrata API endpoint with deterministic pagination (`$order: ':id'`) and updates `input_files/ny_brfss_health_indicators_raw.csv`. +3. `stat_var_processor.py` executes to regenerate the output CSV, TMCF, and MCF artifacts. +4. `validator.py` enforces validation rules from `validation_config.json` before publication. --- -## 4. Execution Workflow +## 4. Script Execution Details + +To run the import pipeline, execute the processing scripts as detailed below. -### Step 1: Download Raw Health Data -To fetch the full historical survey data: +### Download the Data + +This script downloads the multi-year health indicators dataset from the NYSDOH Socrata API into `input_files/`: + +**Usage**: ```bash python3 download.py ``` -This queries the NYSDOH Socrata API for all 17,700 records across all 75 indicators and 5 survey waves, writing directly to `input_files/ny_brfss_health_indicators_raw.csv`. -### Step 2: Generate StatVar Observations & TMCF -Run `stat_var_processor.py` directly on the raw dataset from within the import directory: +### Process the Data + +This script processes the raw input file to generate the final `ny_brfss_health_indicators_output.csv` file, `ny_brfss_health_indicators_output.tmcf` template, and supporting MCF files. + +**Usage**: ```bash python3 ../../../tools/statvar_importer/stat_var_processor.py \ --input_data="input_files/ny_brfss_health_indicators_raw.csv" \ @@ -85,8 +76,11 @@ python3 ../../../tools/statvar_importer/stat_var_processor.py \ --output_counters=counters/ny_brfss_health_indicators_counters.csv ``` -### Step 3: Run Sample Integration Test -Verify property-value mapping against the sample test fixtures: +### Run Sample Integration Test + +This script verifies property-value mapping against the test data fixtures: + +**Usage**: ```bash python3 ../../../tools/statvar_importer/stat_var_processor.py \ --input_data="test_data/sample_input.csv" \ @@ -96,19 +90,16 @@ python3 ../../../tools/statvar_importer/stat_var_processor.py \ --output_path=test_data/sample_expected_output ``` -### Validation Configuration - -Validation is configured in `validation_config.json`: -* `check_deleted_records_percent`: Strictly enforces a historical deletion average threshold of `0.1%`. Note that on the initial import run, this rule expectedly reports missing differ summary because there is no prior version in prod GCS to diff against; it is configured to safeguard future recurring refreshes. Per consensus on initial imports, golden regression files are omitted from initial submission. -* `check_max_date_freshness`: Enforces date freshness (`CAST(MaxDate AS INTEGER) >= 2024`) across active recurring health indicator StatVars (e.g. Diabetes, Obesity, Smoking, Asthma, Hypertension, Cardiovascular Disease, Depression, Dental Visits, Routine Checkups) to ensure the latest published survey wave is always captured. -* `check_import_max_date_freshness`: Enforces that the overall import contains observations through at least 2024 (`CAST(max_date AS INTEGER) >= 2024`). - --- -## 5. Summary Results +### Step 5: Validate the Output Files -* **Observations Generated**: **13,479** `StatVarObservation` records. -* **Geographies Resolved**: **63 unique entities** (all 62 NY counties `geoId/36001` - `geoId/36123` plus New York City `geoId/3651000`). -* **Non-County Regions Dropped**: **3,081 records** (sub-state DSRIP regions, Rest of State, and Statewide rows cleanly dropped by PV map). -* **Statistical Variables Generated**: **75 unique variables** (20 matched to existing canonical DCIDs, 55 provisional StatVars, 56 schema enums/properties). +This command validates the generated files for formatting and semantic consistency before ingestion. +**Usage**: +```bash +java -jar /path/to/datacommons-import-tool.jar lint -d 'output_files/' +``` +This step ensures that the generated artifacts are ready for ingestion into Data Commons. + +--- diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/manifest.json b/statvar_imports/us_newyork/ny_brfss_health_indicators/manifest.json index d0d477f7a3..b74df4dbd4 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/manifest.json +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/manifest.json @@ -20,10 +20,11 @@ ], "source_files": [ "input_files/*.csv", - "counters/*.csv" + "counters/*.csv", + "validation_config.json" ], "cron_schedule": "0 0 1 8 *", "validation_config_file": "validation_config.json" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/validation_config.json b/statvar_imports/us_newyork/ny_brfss_health_indicators/validation_config.json index 9c13cbe0a8..b71f61d93f 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/validation_config.json +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/validation_config.json @@ -29,3 +29,4 @@ } ] } + From d90dcaa0f92737a9e7b71a20be264213c962d55a Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Fri, 11 Sep 2026 07:40:04 +0000 Subject: [PATCH 07/11] crafixes --- .../ny_brfss_health_indicators/README.md | 15 --------------- .../ny_brfss_health_indicators/download.py | 4 ++++ 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md b/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md index 880eb7effa..469faa31cf 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md @@ -21,7 +21,6 @@ The import process involves querying the NYSDOH Socrata API and running a proces * `ny_brfss_health_indicators_metadata.csv`: Configuration file for the data processing script specifying column mappings, header row offset, and provenance URL. * `ny_brfss_health_indicators_pv_map.csv`: Property-value mapping file used by the processor to map indicators and county locations to Data Commons entities. * `validation_config.json`: Configuration defining historical deletion and date freshness validation rules. - * `schema.mcf`: Consolidated provisional StatVars (55) and schema nodes (56) for Piper CL ingestion. * `test_data/`: Sample input data and expected output files for integration testing. * **Transformation pipeline**: @@ -89,17 +88,3 @@ python3 ../../../tools/statvar_importer/stat_var_processor.py \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ --output_path=test_data/sample_expected_output ``` - ---- - -### Step 5: Validate the Output Files - -This command validates the generated files for formatting and semantic consistency before ingestion. - -**Usage**: -```bash -java -jar /path/to/datacommons-import-tool.jar lint -d 'output_files/' -``` -This step ensures that the generated artifacts are ready for ingestion into Data Commons. - ---- diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py b/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py index a180cfd6a8..3619561c40 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py @@ -67,6 +67,10 @@ def atomic_to_csv(df: pd.DataFrame, target_path: str) -> None: """Writes a DataFrame to a CSV file atomically via a temporary file.""" temp_path = f'{target_path}.tmp' df.to_csv(temp_path, index=False) + if not (os.path.exists(temp_path) and os.path.getsize(temp_path) > 0): + raise RuntimeError( + f'Atomic write failed: {temp_path} is empty or missing.' + ) os.replace(temp_path, target_path) From f8c457371c1d46b25251d0cf8dc3c8ba93bfe73d Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Fri, 11 Sep 2026 09:14:25 +0000 Subject: [PATCH 08/11] crafixes --- .../ny_brfss_health_indicators/download.py | 4 +--- .../ny_brfss_health_indicators_pv_map.csv | 11 +++++------ .../validation_config.json | 13 ++----------- 3 files changed, 8 insertions(+), 20 deletions(-) diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py b/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py index 3619561c40..575a5c3adc 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py @@ -104,9 +104,7 @@ def download_health_indicators(endpoint: str, break records.extend(chunk) logging.info('Fetched %d records (offset %d).', len(chunk), offset) - if len(chunk) < limit: - break - offset += limit + offset += len(chunk) if not records: logging.error('Health Data NY API returned 0 records.') diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_pv_map.csv b/statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_pv_map.csv index ddef13a5c7..f27299cd73 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_pv_map.csv +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_pv_map.csv @@ -3,7 +3,6 @@ years,observationDate,{@Data} 2013-14,observationDate,2014 region_county,observationAbout,{@Data} unadjusted_rate,value,{Number} -DSRIP region,#IgnoreRow,True NYS Exclusive of NYC,#IgnoreRow,True Statewide,#IgnoreRow,True REGION: Capital Region,#IgnoreRow,True @@ -34,24 +33,24 @@ Asthma Ever,populationType,Person,measuredProperty,percent,healthOutcome,AsthmaE Binge Drinking,dcid,Percent_Person_BingeDrinking,populationType,Person,measuredProperty,percent,healthBehavior,BingeDrinking,unit,Percent Binge or Heavy Drinking,populationType,Person,measuredProperty,percent,healthBehavior,BingeOrHeavyDrinking,unit,Percent Breast Cancer Screening,dcid,Percent_Person_50To74Years_Female_ReceivedMammography,populationType,Person,gender,Female,age,Years50To74,measuredProperty,percent,healthPrevention,Mammography,unit,Percent -Breast Cancer Screening Low Income 50-74,populationType,Person,gender,Female,age,Years50To74,income,USDollar0To24999,measuredProperty,percent,healthPrevention,Mammography,unit,Percent +Breast Cancer Screening Low Income 50-74,populationType,Person,gender,Female,age,Years50To74,income,[USDollar 0 24999],measuredProperty,percent,healthPrevention,Mammography,unit,Percent Cardiovascular Disease Disability,populationType,Person,disabilityStatus,AnyDisability,measuredProperty,percent,healthOutcome,CardiovascularDisease,unit,Percent Cervical Cancer Screening,dcid,Percent_Person_21To65Years_Female_ReceivedCervicalCancerScreening,populationType,Person,gender,Female,age,Years21To65,measuredProperty,percent,healthPrevention,CervicalCancerScreening,unit,Percent Cervical Cancer Screening (2012 guidelines),populationType,Person,gender,Female,age,Years21To65,measuredProperty,percent,healthPrevention,CervicalCancerScreening2012Guidelines,unit,Percent Cervical Cancer Screening (2012 guidelines) 21-29,populationType,Person,gender,Female,age,Years21To29,measuredProperty,percent,healthPrevention,CervicalCancerScreening2012Guidelines,unit,Percent Cervical Cancer Screening (pre-2012 guidelines),populationType,Person,gender,Female,age,Years21To65,measuredProperty,percent,healthPrevention,CervicalCancerScreeningPre2012Guidelines,unit,Percent -Cervical Cancer Screening (pre-2012 guidelines) Low Income,populationType,Person,gender,Female,age,Years21To65,income,USDollar0To24999,measuredProperty,percent,healthPrevention,CervicalCancerScreeningPre2012Guidelines,unit,Percent +Cervical Cancer Screening (pre-2012 guidelines) Low Income,populationType,Person,gender,Female,age,Years21To65,income,[USDollar 0 24999],measuredProperty,percent,healthPrevention,CervicalCancerScreeningPre2012Guidelines,unit,Percent Cholesterol Checked,dcid,Percent_Person_ReceivedCholesterolScreening,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthPrevention,CholesterolScreening,unit,Percent Chronic Disease Self-Management,populationType,Person,measuredProperty,percent,healthBehavior,ChronicDiseaseSelfManagement,unit,Percent Colorectal Cancer Screening,dcid,Percent_Person_50To75Years_ReceivedColorectalCancerScreening,populationType,Person,age,Years50To75,measuredProperty,percent,healthPrevention,ColorectalCancerScreening,unit,Percent Colorectal Cancer Screening 50-64,populationType,Person,age,Years50To64,measuredProperty,percent,healthPrevention,ColorectalCancerScreening,unit,Percent -Colorectal Cancer Screening Low Income,populationType,Person,age,Years50To75,income,USDollar0To24999,measuredProperty,percent,healthPrevention,ColorectalCancerScreening,unit,Percent +Colorectal Cancer Screening Low Income,populationType,Person,age,Years50To75,income,[USDollar 0 24999],measuredProperty,percent,healthPrevention,ColorectalCancerScreening,unit,Percent Current Cannabis Use,populationType,Person,measuredProperty,percent,healthBehavior,CannabisUse,unit,Percent Current Medical Cannabis Use,populationType,Person,measuredProperty,percent,healthBehavior,MedicalCannabisUse,unit,Percent Current Smoking,dcid,Percent_Person_Smoking,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthBehavior,Smoking,unit,Percent Current Smoking 18-24,populationType,Person,age,Years18To24,measuredProperty,percent,healthBehavior,Smoking,unit,Percent Current Smoking Disability,populationType,Person,disabilityStatus,AnyDisability,measuredProperty,percent,healthBehavior,Smoking,unit,Percent -Current Smoking Low Income,populationType,Person,income,USDollar0To24999,measuredProperty,percent,healthBehavior,Smoking,unit,Percent +Current Smoking Low Income,populationType,Person,income,[USDollar 0 24999],measuredProperty,percent,healthBehavior,Smoking,unit,Percent Current Smoking Poor Mental Health,populationType,Person,healthStatus,PoorMentalHealth,measuredProperty,percent,healthBehavior,Smoking,unit,Percent DWI,populationType,Person,measuredProperty,percent,healthBehavior,DrivingWhileIntoxicated,unit,Percent Diabetes and Prediabetes Testing,populationType,Person,measuredProperty,percent,healthCareUtilization,DiabetesOrPrediabetesTesting,unit,Percent @@ -77,7 +76,7 @@ No Fruits or Vegetables,populationType,Person,measuredProperty,percent,healthBeh No Medical Care Due to Cost,populationType,Person,measuredProperty,percent,healthCareAccessStatus,DidNotReceiveCareDueToCost,unit,Percent Obesity Disability,populationType,Person,disabilityStatus,AnyDisability,measuredProperty,percent,healthBehavior,Obesity,unit,Percent Obesity Limitation Status,populationType,Person,healthStatus,WithLimitation,measuredProperty,percent,healthBehavior,Obesity,unit,Percent -Obesity Low Income,populationType,Person,income,USDollar0To24999,measuredProperty,percent,healthBehavior,Obesity,unit,Percent +Obesity Low Income,populationType,Person,income,[USDollar 0 24999],measuredProperty,percent,healthBehavior,Obesity,unit,Percent PSA Screening,populationType,Person,gender,Male,age,Years55To69,measuredProperty,percent,healthCareUtilization,ProstateSpecificAntigenScreeningSharedDecision,unit,Percent Pneumonia Shot 65+,populationType,Person,age,Years65Onwards,measuredProperty,percent,healthCareUtilization,PneumococcalVaccineEver,unit,Percent Poor General Health,dcid,Percent_Person_18OrMoreYears_WithPoorGeneralHealth,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthOutcome,PoorGeneralHealth,unit,Percent diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/validation_config.json b/statvar_imports/us_newyork/ny_brfss_health_indicators/validation_config.json index b71f61d93f..e3e8af52fe 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/validation_config.json +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/validation_config.json @@ -11,20 +11,11 @@ }, { "rule_id": "check_max_date_freshness", - "description": "Verify MaxDate is at least 2024 for recurring health indicators", + "description": "Verify MaxDate is within allowable publication lag for recurring health indicators", "validator": "SQL_VALIDATOR", "params": { "query": "SELECT StatVar, MaxDate FROM stats WHERE StatVar IN ('Percent_Person_WithDiabetes', 'Percent_Person_Obesity', 'Percent_Person_Smoking', 'Percent_Person_WithAsthma', 'Percent_Person_WithArthritis', 'Percent_Person_WithHighBloodPressure', 'Percent_Person_WithCardiovascularDisease', 'Percent_Person_WithChronicObstructivePulmonaryDisease', 'Percent_Person_18OrMoreYears_WithDepression', 'Percent_Person_18OrMoreYears_WithPoorGeneralHealth', 'Percent_Person_BingeDrinking', 'Percent_Person_LeisureTimePhysicalActivity', 'Percent_Person_ReceivedDentalVisit', 'Percent_Person_WithPrediabetes', 'Percent_Person_18To64Years_WithHealthInsurance', 'Percent_Person_18OrMoreYears_WithAnyDisability', 'Percent_Person_Years18To64_RoutineCheckupWithinPastYear', 'Percent_Person_50To74Years_Female_ReceivedMammography')", - "condition": "CAST(MaxDate AS INTEGER) >= 2024" - } - }, - { - "rule_id": "check_import_max_date_freshness", - "description": "Verify the import contains latest survey wave observations through at least 2024", - "validator": "SQL_VALIDATOR", - "params": { - "query": "SELECT MAX(MaxDate) AS max_date FROM stats", - "condition": "CAST(max_date AS INTEGER) >= 2024" + "condition": "CAST(MaxDate AS INTEGER) >= (EXTRACT(YEAR FROM CURRENT_DATE) - 2)" } } ] From 32dd32ad9e30b40e50af85a8b22b4ffc78de183c Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Fri, 11 Sep 2026 12:24:43 +0000 Subject: [PATCH 09/11] crafixes --- .../ny_brfss_health_indicators/README.md | 2 +- .../ny_brfss_health_indicators/download.py | 44 +++++++++---------- .../ny_brfss_health_indicators/manifest.json | 2 +- .../validation_config.json | 2 +- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md b/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md index 469faa31cf..471016d1bc 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/README.md @@ -31,7 +31,7 @@ The import process involves querying the NYSDOH Socrata API and running a proces * **Data Quality Checks**: * The `dc_generated/` directory contains `report.json` and `summary_report.csv`, which provide validation and summary statistics for the generated data. - * Automated validation via `validator.py` evaluates the output against `validation_config.json` enforcing the historical deletion threshold (<= 0.1%) and date freshness (`CAST(MaxDate AS INTEGER) >= 2024`). + * Automated validation via `validator.py` evaluates the output against `validation_config.json` enforcing the historical deletion threshold (<= 0.1%) and date freshness (`CAST(MaxDate AS INTEGER) >= (EXTRACT(YEAR FROM CURRENT_DATE) - 3)`). --- diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py b/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py index 575a5c3adc..00037e3b53 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/download.py @@ -77,34 +77,34 @@ def atomic_to_csv(df: pd.DataFrame, target_path: str) -> None: def download_health_indicators(endpoint: str, output_dir: str) -> tuple[int, list[str]]: """Downloads all health indicators for all NY counties and regions.""" - session = create_session() os.makedirs(output_dir, exist_ok=True) records = [] offset = 0 limit = 50000 - while True: - params = { - '$limit': limit, - '$offset': offset, - '$order': ':id', - } - logging.info('GET %s with params %s', endpoint, params) - resp = session.get(endpoint, params=params, timeout=60) - if resp.status_code != 200: - logging.error('Health Data NY API returned HTTP %d: %s', - resp.status_code, resp.text[:200]) - raise RuntimeError( - f'Health Data NY API returned HTTP {resp.status_code}: {resp.text[:200]}' - ) - - chunk = resp.json() - if not chunk: - break - records.extend(chunk) - logging.info('Fetched %d records (offset %d).', len(chunk), offset) - offset += len(chunk) + with create_session() as session: + while True: + params = { + '$limit': limit, + '$offset': offset, + '$order': ':id', + } + logging.info('GET %s with params %s', endpoint, params) + resp = session.get(endpoint, params=params, timeout=60) + if resp.status_code != 200: + logging.error('Health Data NY API returned HTTP %d: %s', + resp.status_code, resp.text[:200]) + raise RuntimeError( + f'Health Data NY API returned HTTP {resp.status_code}: {resp.text[:200]}' + ) + + chunk = resp.json() + if not chunk: + break + records.extend(chunk) + logging.info('Fetched %d records (offset %d).', len(chunk), offset) + offset += len(chunk) if not records: logging.error('Health Data NY API returned 0 records.') diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/manifest.json b/statvar_imports/us_newyork/ny_brfss_health_indicators/manifest.json index b74df4dbd4..5be4264a3a 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/manifest.json +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/manifest.json @@ -9,7 +9,7 @@ "provenance_description": "The Behavioral Risk Factor Surveillance System (BRFSS) Health Indicators dataset from the New York State Department of Health provides annual estimates of chronic disease prevalence across all counties in New York.", "scripts": [ "download.py", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=\"input_files/ny_brfss_health_indicators_raw.csv\" --pv_map=ny_brfss_health_indicators_pv_map.csv --config_file=ny_brfss_health_indicators_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ny_brfss_health_indicators_output --output_counters=counters/ny_brfss_health_indicators_counters.csv" + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ny_brfss_health_indicators_raw.csv --pv_map=ny_brfss_health_indicators_pv_map.csv --config_file=ny_brfss_health_indicators_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ny_brfss_health_indicators_output --output_counters=counters/ny_brfss_health_indicators_counters.csv" ], "import_inputs": [ { diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/validation_config.json b/statvar_imports/us_newyork/ny_brfss_health_indicators/validation_config.json index e3e8af52fe..9916221389 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/validation_config.json +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/validation_config.json @@ -15,7 +15,7 @@ "validator": "SQL_VALIDATOR", "params": { "query": "SELECT StatVar, MaxDate FROM stats WHERE StatVar IN ('Percent_Person_WithDiabetes', 'Percent_Person_Obesity', 'Percent_Person_Smoking', 'Percent_Person_WithAsthma', 'Percent_Person_WithArthritis', 'Percent_Person_WithHighBloodPressure', 'Percent_Person_WithCardiovascularDisease', 'Percent_Person_WithChronicObstructivePulmonaryDisease', 'Percent_Person_18OrMoreYears_WithDepression', 'Percent_Person_18OrMoreYears_WithPoorGeneralHealth', 'Percent_Person_BingeDrinking', 'Percent_Person_LeisureTimePhysicalActivity', 'Percent_Person_ReceivedDentalVisit', 'Percent_Person_WithPrediabetes', 'Percent_Person_18To64Years_WithHealthInsurance', 'Percent_Person_18OrMoreYears_WithAnyDisability', 'Percent_Person_Years18To64_RoutineCheckupWithinPastYear', 'Percent_Person_50To74Years_Female_ReceivedMammography')", - "condition": "CAST(MaxDate AS INTEGER) >= (EXTRACT(YEAR FROM CURRENT_DATE) - 2)" + "condition": "CAST(MaxDate AS INTEGER) >= (EXTRACT(YEAR FROM CURRENT_DATE) - 3)" } } ] From edef19a5918c2fea8c38075620be03f90ded3273 Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Mon, 14 Sep 2026 08:55:29 +0000 Subject: [PATCH 10/11] internal review suggestions --- .../ny_brfss_health_indicators_pv_map.csv | 52 ++++++++++++++----- .../test_data/sample_expected_output.tmcf | 7 --- .../sample_expected_output_stat_vars.mcf | 16 ------ ...ample_expected_output_stat_vars_schema.mcf | 10 ---- 4 files changed, 39 insertions(+), 46 deletions(-) delete mode 100644 statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output.tmcf delete mode 100644 statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output_stat_vars.mcf delete mode 100644 statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output_stat_vars_schema.mcf diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_pv_map.csv b/statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_pv_map.csv index f27299cd73..bba03dfce7 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_pv_map.csv +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/ny_brfss_health_indicators_pv_map.csv @@ -3,19 +3,45 @@ years,observationDate,{@Data} 2013-14,observationDate,2014 region_county,observationAbout,{@Data} unadjusted_rate,value,{Number} -NYS Exclusive of NYC,#IgnoreRow,True -Statewide,#IgnoreRow,True -REGION: Capital Region,#IgnoreRow,True -REGION: Central NY,#IgnoreRow,True -REGION: Finger Lakes,#IgnoreRow,True -REGION: Long Island,#IgnoreRow,True -REGION: Mid-Hudson,#IgnoreRow,True -REGION: Mohawk Valley,#IgnoreRow,True -REGION: New York City,#IgnoreRow,True -REGION: North Country,#IgnoreRow,True -REGION: Southern Tier,#IgnoreRow,True -REGION: Tug Hill Seaway,#IgnoreRow,True -REGION: Western NY,#IgnoreRow,True +NYS Exclusive of NYC,#IgnoreRow,True,#ignore,non-county regional aggregate +Statewide,#IgnoreRow,True,#ignore,statewide aggregate +REGION: Capital Region,#IgnoreRow,True,#ignore,non-county regional aggregate +REGION: Central NY,#IgnoreRow,True,#ignore,non-county regional aggregate +REGION: Finger Lakes,#IgnoreRow,True,#ignore,non-county regional aggregate +REGION: Long Island,#IgnoreRow,True,#ignore,non-county regional aggregate +REGION: Mid-Hudson,#IgnoreRow,True,#ignore,non-county regional aggregate +REGION: Mohawk Valley,#IgnoreRow,True,#ignore,non-county regional aggregate +REGION: New York City,#IgnoreRow,True,#ignore,non-county regional aggregate +REGION: North Country,#IgnoreRow,True,#ignore,non-county regional aggregate +REGION: Southern Tier,#IgnoreRow,True,#ignore,non-county regional aggregate +REGION: Tug Hill Seaway,#IgnoreRow,True,#ignore,non-county regional aggregate +REGION: Western NY,#IgnoreRow,True,#ignore,non-county regional aggregate +sample_size_n,#ignore,survey sample size not a target health indicator,value,{@Data} +est_count,#ignore,weighted population estimate count,value,{@Data} +est_denominator,#ignore,weighted denominator population,value,{@Data} +cl_95_lower_unadjusted,#ignore,95% confidence interval lower bound,value,{@Data} +cl_95_upper_unadjusted,#ignore,95% confidence interval upper bound,value,{@Data} +adjusted_rate,#ignore,age-adjusted rate omitted in favor of unadjusted rate,value,{@Data} +cl_95_lower_adjusted,#ignore,95% confidence interval lower bound for adjusted rate,value,{@Data} +cl_95_upper_adjusted,#ignore,95% confidence interval upper bound for adjusted rate,value,{@Data} +geography,#ignore,redundant geography level label,value,{@Data} +health_domain,#ignore,category label redundant with StatVar,value,{@Data} +health_indicator,#ignore,verbose description redundant with short name,value,{@Data} +prevention_agenda_pa,#ignore,NYSDOH program tracking flag,value,{@Data} +ship,#ignore,NYSDOH program tracking flag,value,{@Data} +dsrip,#ignore,NYSDOH program tracking flag,value,{@Data} +chirs,#ignore,NYSDOH program tracking flag,value,{@Data} +fips,#ignore,redundant with region_county mapping,value,{@Data} +health_indicator_sort_key,#ignore,internal display sort key,value,{@Data} +dsrip_region_sort_key,#ignore,internal display sort key,value,{@Data} +county_sort_key,#ignore,internal display sort key,value,{@Data} +location,#ignore,WKT centroid coordinate redundant with place DCID,value,{@Data} +notes,#ignore,data suppression note,value,{@Data} +pa_rate_type,#ignore,rate type metadata,value,{@Data} +ship_rate_type,#ignore,rate type metadata,value,{@Data} +dsrip_rate_type,#ignore,rate type metadata,value,{@Data} +chirs_rate_type,#ignore,rate type metadata,value,{@Data} +upstate_downstate_sort_key,#ignore,internal display sort key,value,{@Data} Diabetes,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthOutcome,Diabetes,unit,Percent Prediabetes,populationType,Person,measuredProperty,percent,healthOutcome,Prediabetes,unit,Percent Asthma,populationType,Person,age,Years18Onwards,measuredProperty,percent,healthOutcome,Asthma,unit,Percent diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output.tmcf b/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output.tmcf deleted file mode 100644 index b885d89ede..0000000000 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output.tmcf +++ /dev/null @@ -1,7 +0,0 @@ -Node: E:sample_expected_output->E0 -observationAbout: C:sample_expected_output->observationAbout -observationDate: C:sample_expected_output->observationDate -value: C:sample_expected_output->value -variableMeasured: C:sample_expected_output->variableMeasured -unit: Percent -typeOf: dcs:StatVarObservation diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output_stat_vars.mcf b/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output_stat_vars.mcf deleted file mode 100644 index ba97cb6730..0000000000 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output_stat_vars.mcf +++ /dev/null @@ -1,16 +0,0 @@ -# Auto generated using command: "../../../tools/statvar_importer/stat_var_processor.py --input_data=test_data/sample_input.csv --pv_map=ny_brfss_health_indicators_pv_map.csv --config_file=ny_brfss_health_indicators_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=test_data/sample_expected_output" on 2026-09-09 08:44:05.973952 - -Node: dcid:Percent_Person_WithCardiovascularDisease -typeOf: dcid:StatisticalVariable -populationType: dcid:Person -measuredProperty: dcid:percent -statType: dcid:measuredValue -healthOutcome: dcid:CardiovascularDisease - -Node: dcid:Percent_Person_WithPrediabetes -typeOf: dcid:StatisticalVariable -populationType: dcid:Person -measuredProperty: dcid:percent -statType: dcid:measuredValue -healthOutcome: dcid:Prediabetes - diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output_stat_vars_schema.mcf b/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output_stat_vars_schema.mcf deleted file mode 100644 index cd434872ce..0000000000 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/test_data/sample_expected_output_stat_vars_schema.mcf +++ /dev/null @@ -1,10 +0,0 @@ -Node: dcid:Prediabetes -typeOf: dcid:HealthOutcomeEnum -name: "Prediabetes" -isProvisional: dcs:True - -Node: dcid:healthOutcome -typeOf: dcs:Property -name: "healthOutcome" -rangeIncludes: dcid:MedicalConditionIncidentTypeEnum - From a80b5e09b23bc6767a642e69b672ca140d414b9b Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Mon, 14 Sep 2026 09:01:22 +0000 Subject: [PATCH 11/11] cron schedule change --- .../us_newyork/ny_brfss_health_indicators/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statvar_imports/us_newyork/ny_brfss_health_indicators/manifest.json b/statvar_imports/us_newyork/ny_brfss_health_indicators/manifest.json index 5be4264a3a..54b5ae876f 100644 --- a/statvar_imports/us_newyork/ny_brfss_health_indicators/manifest.json +++ b/statvar_imports/us_newyork/ny_brfss_health_indicators/manifest.json @@ -23,7 +23,7 @@ "counters/*.csv", "validation_config.json" ], - "cron_schedule": "0 0 1 8 *", + "cron_schedule": "0 0 1 * *", "validation_config_file": "validation_config.json" } ]