From 59ec707a4f9071bd95bb49cc41934f3f2437cee9 Mon Sep 17 00:00:00 2001 From: Nick Eubank Date: Wed, 7 Oct 2026 20:11:56 -0400 Subject: [PATCH] Add a vintage argument to shapefile_urls() `shapefile_urls()` has been pinned to the 2010 TIGER/Line files since it was written. Add a `vintage` argument so callers can ask for the 2020 vintage instead; an unsupported vintage raises ValueError, and states without a FIPS code still return None. The default stays 2010, since changing it outright would break existing callers. Calling without a `vintage` raises a DeprecationWarning saying the default becomes 2020 in us 5.0; passing either vintage explicitly opts out of the warning. The states CLI passes the default explicitly so it tracks the library default without warning its own users. The 2020 vintage isn't a drop-in path swap: the Census Bureau only publishes nationwide files for the cd, county, state and zcta layers, so those URLs are the same for every state. 2020 census blocks live in TABBLOCK20 (TIGER2020/TABBLOCK holds reissued 2010 blocks) and 2020 ZCTAs in ZCTA520; cd is the 116th Congress, the only one in this vintage. Replace the permanently skipped test_head with offline tests covering the default vintage, the deprecation warning, the exact URLs for both vintages, and all 56 states and territories, plus `network`-marked tests that HEAD every generated URL against the Census Bureau. Those use urllib rather than requests, so no new dependency, and are deselected by default since they make a few hundred requests. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 7 ++ README.md | 29 +++++++- pyproject.toml | 6 ++ us/cli/states.py | 4 +- us/states.py | 68 ++++++++++++++---- us/tests/test_us.py | 167 +++++++++++++++++++++++++++++++++++++++++--- 6 files changed, 254 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e755df7..6ae8d23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## Unreleased + +* add a `vintage` argument to `shapefile_urls()`, so URLs can be generated for either the 2010 or the 2020 TIGER/Line vintage. In the 2020 vintage the Census Bureau only publishes nationwide files for the `cd`, `county`, `state` and `zcta` layers, so those URLs are the same for every state. +* `vintage` still defaults to 2010, but that default will change to 2020 in the 5.0 release. Calling `shapefile_urls()` without a `vintage` now raises a `DeprecationWarning`; pass `vintage=2010` to keep the current URLs, or `vintage=2020` to opt in to the new default now. +* add `network`-marked tests that check the generated shapefile URLs against the Census Bureau, deselected by default + + ## 4.0.0 * add counties, thanks to [Ray Kiddy](https://github.com/rkiddy) diff --git a/README.md b/README.md index b1c738d..80da211 100644 --- a/README.md +++ b/README.md @@ -200,10 +200,10 @@ no fallback at all). ### Shapefiles -You want shapefiles too? As long as you want 2010 shapefiles, we've gotcha covered. +You want shapefiles too? We've gotcha covered. ``` ->>> urls = us.states.MD.shapefile_urls() +>>> urls = us.states.MD.shapefile_urls(vintage=2010) >>> sorted(urls.keys()) ['block', 'blockgroup', 'cd', 'county', 'state', 'tract', 'zcta'] >>> urls['block'] @@ -221,6 +221,23 @@ the following regions: * state * zcta +The `vintage` argument selects the TIGER/Line vintage, either 2010 or 2020: + +``` +>>> us.states.MD.shapefile_urls(vintage=2020)['block'] +'https://www2.census.gov/geo/tiger/TIGER2020/TABBLOCK20/tl_2020_24_tabblock20.zip' +``` + +**Deprecation:** `vintage` currently defaults to 2010, but in us 5.0 the +default will change to 2020. Calling `shapefile_urls()` without a `vintage` +raises a `DeprecationWarning` until then. Pass `vintage=2010` to keep today's +URLs, or `vintage=2020` to opt in to the new default now. + +Note that the Census Bureau stopped publishing per-state files for some layers +after 2010. In the 2020 vintage the `cd`, `county`, `state` and `zcta` URLs +point at nationwide files, so they're identical for every state. The `block` +URLs are for 2020 census blocks (`TABBLOCK20`), and `cd` is the 116th Congress. + ### Mappings @@ -327,6 +344,14 @@ uv sync uv run pytest ``` +Tests marked `network` check the generated shapefile URLs against the Census +Bureau. They're deselected by default because they need network access and +make a few hundred requests. To run them: + +``` +uv run pytest -m network +``` + ## Changelog diff --git a/pyproject.toml b/pyproject.toml index c7f9d7b..8b70cfd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,3 +40,9 @@ dev = [ [tool.ruff] line-length = 120 target-version = "py38" + +[tool.pytest.ini_options] +markers = [ + "network: makes live requests to the Census Bureau (deselected by default, run with `-m network`)", +] +addopts = "-m 'not network'" diff --git a/us/cli/states.py b/us/cli/states.py index c075ad4..5a98022 100644 --- a/us/cli/states.py +++ b/us/cli/states.py @@ -42,7 +42,9 @@ def main(): sys.stdout.write("\n") sys.stdout.write(" shapefiles:\n") - urls = state.shapefile_urls() + # pass the default explicitly: the CLI tracks whatever the library + # default is without tripping its deprecation warning + urls = state.shapefile_urls(vintage=us.states.DEFAULT_SHAPEFILE_VINTAGE) if urls is not None: for region, url in urls.items(): sys.stdout.write(" %s: %s\n" % (region, url)) diff --git a/us/states.py b/us/states.py index cd19149..2e2e278 100644 --- a/us/states.py +++ b/us/states.py @@ -1,5 +1,6 @@ import os import re +import warnings from enum import Enum from typing import Any, Callable, Dict, Iterable, List, Optional, Type from urllib.parse import urljoin @@ -11,6 +12,12 @@ DC_STATEHOOD = bool(os.environ.get("DC_STATEHOOD")) +# TIGER/Line vintages that `State.shapefile_urls()` knows how to build URLs for. +SHAPEFILE_VINTAGES = (2010, 2020) +DEFAULT_SHAPEFILE_VINTAGE = 2010 +# the vintage `shapefile_urls()` will default to in the 5.0 release +FUTURE_DEFAULT_SHAPEFILE_VINTAGE = 2020 + _lookup_cache: Dict[str, "State"] = {} @@ -58,28 +65,63 @@ def __repr__(self) -> str: def __str__(self) -> str: return self.name - def shapefile_urls(self) -> Optional[Dict[str, str]]: + def shapefile_urls(self, vintage: Optional[int] = None) -> Optional[Dict[str, str]]: """Shapefiles are available directly from the US Census Bureau: https://www.census.gov/cgi-bin/geo/shapefiles/index.php + + `vintage` selects the TIGER/Line vintage, either 2010 or 2020. It + currently defaults to 2010, but in the 5.0 release the default will + change to 2020, so calling without a `vintage` raises a + DeprecationWarning. Pass `vintage=2010` to keep today's URLs, or + `vintage=2020` to opt in to the new default now. + + Note that the Census Bureau stopped publishing per-state files for + some layers after 2010. In the 2020 vintage the "cd", "county", + "state" and "zcta" URLs therefore point at nationwide files, which + are identical for every state. """ + if vintage is None: + warnings.warn( + f"shapefile_urls() currently defaults to the {DEFAULT_SHAPEFILE_VINTAGE} TIGER/Line vintage, " + f"but will default to the {FUTURE_DEFAULT_SHAPEFILE_VINTAGE} vintage in us 5.0. " + f"Pass vintage={DEFAULT_SHAPEFILE_VINTAGE} to keep the current URLs, or " + f"vintage={FUTURE_DEFAULT_SHAPEFILE_VINTAGE} to opt in to the new default now.", + DeprecationWarning, + stacklevel=2, + ) + vintage = DEFAULT_SHAPEFILE_VINTAGE + fips = self.fips if not fips: return None - base = "https://www2.census.gov/geo/tiger/TIGER2010/" - urls = { - "tract": urljoin(base, f"TRACT/2010/tl_2010_{fips}_tract10.zip"), - "cd": urljoin(base, f"CD/111/tl_2010_{fips}_cd111.zip"), - "county": urljoin(base, f"COUNTY/2010/tl_2010_{fips}_county10.zip"), - "state": urljoin(base, f"STATE/2010/tl_2010_{fips}_state10.zip"), - "zcta": urljoin(base, f"ZCTA5/2010/tl_2010_{fips}_zcta510.zip"), - "block": urljoin(base, f"TABBLOCK/2010/tl_2010_{fips}_tabblock10.zip"), - "blockgroup": urljoin(base, f"BG/2010/tl_2010_{fips}_bg10.zip"), - } - - return urls + if vintage == 2010: + base = "https://www2.census.gov/geo/tiger/TIGER2010/" + return { + "tract": urljoin(base, f"TRACT/2010/tl_2010_{fips}_tract10.zip"), + "cd": urljoin(base, f"CD/111/tl_2010_{fips}_cd111.zip"), + "county": urljoin(base, f"COUNTY/2010/tl_2010_{fips}_county10.zip"), + "state": urljoin(base, f"STATE/2010/tl_2010_{fips}_state10.zip"), + "zcta": urljoin(base, f"ZCTA5/2010/tl_2010_{fips}_zcta510.zip"), + "block": urljoin(base, f"TABBLOCK/2010/tl_2010_{fips}_tabblock10.zip"), + "blockgroup": urljoin(base, f"BG/2010/tl_2010_{fips}_bg10.zip"), + } + + if vintage == 2020: + base = "https://www2.census.gov/geo/tiger/TIGER2020/" + return { + "tract": urljoin(base, f"TRACT/tl_2020_{fips}_tract.zip"), + "cd": urljoin(base, "CD/tl_2020_us_cd116.zip"), + "county": urljoin(base, "COUNTY/tl_2020_us_county.zip"), + "state": urljoin(base, "STATE/tl_2020_us_state.zip"), + "zcta": urljoin(base, "ZCTA520/tl_2020_us_zcta520.zip"), + "block": urljoin(base, f"TABBLOCK20/tl_2020_{fips}_tabblock20.zip"), + "blockgroup": urljoin(base, f"BG/tl_2020_{fips}_bg.zip"), + } + + raise ValueError(f"unsupported shapefile vintage {vintage!r}, expected one of {SHAPEFILE_VINTAGES}") def lookup( diff --git a/us/tests/test_us.py b/us/tests/test_us.py index 5dc4196..ca5317e 100644 --- a/us/tests/test_us.py +++ b/us/tests/test_us.py @@ -1,4 +1,10 @@ +import os.path import re +import time +import urllib.error +import urllib.request +import warnings +from concurrent.futures import ThreadPoolExecutor from itertools import chain import jellyfish @@ -262,20 +268,159 @@ def test_cli_lookup(capsys, monkeypatch): # shapefiles +SHAPEFILE_REGIONS = {"block", "blockgroup", "cd", "county", "state", "tract", "zcta"} -@pytest.mark.skip -def test_head(): - # `requests` is intentionally not declared as a dependency; this test - # is permanently skipped and the import is dead code for ty. - import requests # ty: ignore[unresolved-import] +# layers the Census Bureau only publishes as a single nationwide file in the +# 2020 vintage, so the URL is the same no matter which state you ask for +NATIONWIDE_2020_REGIONS = {"cd", "county", "state", "zcta"} +TIGER2020 = "https://www2.census.gov/geo/tiger/TIGER2020/" +TIGER2010 = "https://www2.census.gov/geo/tiger/TIGER2010/" + + +def test_shapefile_urls_default_to_2010(): + # the default is still 2010 until the 5.0 release + assert us.states.DEFAULT_SHAPEFILE_VINTAGE == 2010 + + with pytest.warns(DeprecationWarning): + urls = us.states.MD.shapefile_urls() + + assert urls == { + "tract": TIGER2010 + "TRACT/2010/tl_2010_24_tract10.zip", + "cd": TIGER2010 + "CD/111/tl_2010_24_cd111.zip", + "county": TIGER2010 + "COUNTY/2010/tl_2010_24_county10.zip", + "state": TIGER2010 + "STATE/2010/tl_2010_24_state10.zip", + "zcta": TIGER2010 + "ZCTA5/2010/tl_2010_24_zcta510.zip", + "block": TIGER2010 + "TABBLOCK/2010/tl_2010_24_tabblock10.zip", + "blockgroup": TIGER2010 + "BG/2010/tl_2010_24_bg10.zip", + } + + +def test_shapefile_urls_default_matches_explicit_2010(): + with pytest.warns(DeprecationWarning): + default_urls = us.states.MD.shapefile_urls() + + assert default_urls == us.states.MD.shapefile_urls(vintage=2010) + + +@pytest.mark.parametrize("state", [us.states.MD, us.states.PR]) +def test_shapefile_urls_default_warns_about_the_5_0_change(state): + with pytest.warns(DeprecationWarning) as record: + state.shapefile_urls() + + assert len(record) == 1 + + message = str(record[0].message) + assert "shapefile_urls()" in message + assert "2010" in message + assert "2020" in message + assert "5.0" in message + # the warning should tell callers how to pin either vintage + assert "vintage=2010" in message + assert "vintage=2020" in message + + # stacklevel should point the warning at the caller, not at us/states.py + assert os.path.basename(record[0].filename) == os.path.basename(__file__) + + +def test_shapefile_urls_explicit_vintage_does_not_warn(): + # passing a vintage is how callers opt out of the deprecation warning + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + + for vintage in us.states.SHAPEFILE_VINTAGES: + assert us.states.MD.shapefile_urls(vintage=vintage) is not None + + +def test_shapefile_urls_2020(): + # the vintage that will become the default in 5.0 + assert us.states.FUTURE_DEFAULT_SHAPEFILE_VINTAGE == 2020 + + urls = us.states.MD.shapefile_urls(vintage=2020) + assert urls == { + "tract": TIGER2020 + "TRACT/tl_2020_24_tract.zip", + "cd": TIGER2020 + "CD/tl_2020_us_cd116.zip", + "county": TIGER2020 + "COUNTY/tl_2020_us_county.zip", + "state": TIGER2020 + "STATE/tl_2020_us_state.zip", + "zcta": TIGER2020 + "ZCTA520/tl_2020_us_zcta520.zip", + "block": TIGER2020 + "TABBLOCK20/tl_2020_24_tabblock20.zip", + "blockgroup": TIGER2020 + "BG/tl_2020_24_bg.zip", + } + + +@pytest.mark.parametrize("vintage", us.states.SHAPEFILE_VINTAGES) +def test_shapefile_urls_cover_every_state_and_territory(vintage): + for state in us.STATES_AND_TERRITORIES: + urls = state.shapefile_urls(vintage=vintage) + assert urls is not None, state + assert set(urls) == SHAPEFILE_REGIONS, state + + for region, url in urls.items(): + assert url.startswith(f"https://www2.census.gov/geo/tiger/TIGER{vintage}/"), url + assert url.endswith(".zip"), url + if vintage == 2020 and region in NATIONWIDE_2020_REGIONS: + assert "_us_" in url + else: + assert f"_{state.fips}_" in url, url + + +def test_shapefile_urls_without_fips(): + # obsolete states have no FIPS code, so there is nothing to build a URL from + for state in us.OBSOLETE: + assert state.fips is None + for vintage in us.states.SHAPEFILE_VINTAGES: + assert state.shapefile_urls(vintage=vintage) is None + + +def test_shapefile_urls_unsupported_vintage(): + with pytest.raises(ValueError, match="unsupported shapefile vintage"): + us.states.MD.shapefile_urls(vintage=2000) + + +def _head_status(url: str, attempts: int = 3) -> int: + """HEAD `url` and return its status code.""" + + request = urllib.request.Request(url, method="HEAD") + + for attempt in range(1, attempts + 1): + try: + with urllib.request.urlopen(request, timeout=60) as resp: + return resp.getcode() + except urllib.error.HTTPError as exc: + # a real answer from the server, including a 404 for a bad URL + return exc.code + except urllib.error.URLError: + # DNS and connection hiccups are common when firing off a few + # hundred requests, so retry before calling the URL broken + if attempt == attempts: + raise + time.sleep(attempt) + + raise AssertionError("unreachable") + + +@pytest.mark.network +@pytest.mark.parametrize("vintage", us.states.SHAPEFILE_VINTAGES) +def test_shapefile_urls_are_live(vintage): + """Check every generated URL against the Census Bureau. + + Deselected by default because it needs network access and makes a few + hundred requests. Run it with `uv run pytest -m network`. + """ + urls = set() for state in us.STATES_AND_TERRITORIES: - urls = state.shapefile_urls() - if urls is None: - continue - for url in urls.values(): - resp = requests.head(url) - assert resp.status_code == 200 + state_urls = state.shapefile_urls(vintage=vintage) + assert state_urls is not None + urls.update(state_urls.values()) + + # the nationwide 2020 layers repeat across states, so check each URL once + unique_urls = sorted(urls) + + with ThreadPoolExecutor(max_workers=4) as pool: + statuses = dict(zip(unique_urls, pool.map(_head_status, unique_urls))) + + broken = {url: status for url, status in statuses.items() if status != 200} + assert not broken, f"{len(broken)} of {len(unique_urls)} URLs did not return 200: {broken}" # counts