diff --git a/CHANGELOG.md b/CHANGELOG.md index e755df7..6ae8d23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## Unreleased + +* add a `vintage` argument to `shapefile_urls()`, so URLs can be generated for either the 2010 or the 2020 TIGER/Line vintage. In the 2020 vintage the Census Bureau only publishes nationwide files for the `cd`, `county`, `state` and `zcta` layers, so those URLs are the same for every state. +* `vintage` still defaults to 2010, but that default will change to 2020 in the 5.0 release. Calling `shapefile_urls()` without a `vintage` now raises a `DeprecationWarning`; pass `vintage=2010` to keep the current URLs, or `vintage=2020` to opt in to the new default now. +* add `network`-marked tests that check the generated shapefile URLs against the Census Bureau, deselected by default + + ## 4.0.0 * add counties, thanks to [Ray Kiddy](https://github.com/rkiddy) diff --git a/README.md b/README.md index b1c738d..80da211 100644 --- a/README.md +++ b/README.md @@ -200,10 +200,10 @@ no fallback at all). ### Shapefiles -You want shapefiles too? As long as you want 2010 shapefiles, we've gotcha covered. +You want shapefiles too? We've gotcha covered. ``` ->>> urls = us.states.MD.shapefile_urls() +>>> urls = us.states.MD.shapefile_urls(vintage=2010) >>> sorted(urls.keys()) ['block', 'blockgroup', 'cd', 'county', 'state', 'tract', 'zcta'] >>> urls['block'] @@ -221,6 +221,23 @@ the following regions: * state * zcta +The `vintage` argument selects the TIGER/Line vintage, either 2010 or 2020: + +``` +>>> us.states.MD.shapefile_urls(vintage=2020)['block'] +'https://www2.census.gov/geo/tiger/TIGER2020/TABBLOCK20/tl_2020_24_tabblock20.zip' +``` + +**Deprecation:** `vintage` currently defaults to 2010, but in us 5.0 the +default will change to 2020. Calling `shapefile_urls()` without a `vintage` +raises a `DeprecationWarning` until then. Pass `vintage=2010` to keep today's +URLs, or `vintage=2020` to opt in to the new default now. + +Note that the Census Bureau stopped publishing per-state files for some layers +after 2010. In the 2020 vintage the `cd`, `county`, `state` and `zcta` URLs +point at nationwide files, so they're identical for every state. The `block` +URLs are for 2020 census blocks (`TABBLOCK20`), and `cd` is the 116th Congress. + ### Mappings @@ -327,6 +344,14 @@ uv sync uv run pytest ``` +Tests marked `network` check the generated shapefile URLs against the Census +Bureau. They're deselected by default because they need network access and +make a few hundred requests. To run them: + +``` +uv run pytest -m network +``` + ## Changelog diff --git a/pyproject.toml b/pyproject.toml index c7f9d7b..8b70cfd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,3 +40,9 @@ dev = [ [tool.ruff] line-length = 120 target-version = "py38" + +[tool.pytest.ini_options] +markers = [ + "network: makes live requests to the Census Bureau (deselected by default, run with `-m network`)", +] +addopts = "-m 'not network'" diff --git a/us/cli/states.py b/us/cli/states.py index c075ad4..5a98022 100644 --- a/us/cli/states.py +++ b/us/cli/states.py @@ -42,7 +42,9 @@ def main(): sys.stdout.write("\n") sys.stdout.write(" shapefiles:\n") - urls = state.shapefile_urls() + # pass the default explicitly: the CLI tracks whatever the library + # default is without tripping its deprecation warning + urls = state.shapefile_urls(vintage=us.states.DEFAULT_SHAPEFILE_VINTAGE) if urls is not None: for region, url in urls.items(): sys.stdout.write(" %s: %s\n" % (region, url)) diff --git a/us/states.py b/us/states.py index cd19149..2e2e278 100644 --- a/us/states.py +++ b/us/states.py @@ -1,5 +1,6 @@ import os import re +import warnings from enum import Enum from typing import Any, Callable, Dict, Iterable, List, Optional, Type from urllib.parse import urljoin @@ -11,6 +12,12 @@ DC_STATEHOOD = bool(os.environ.get("DC_STATEHOOD")) +# TIGER/Line vintages that `State.shapefile_urls()` knows how to build URLs for. +SHAPEFILE_VINTAGES = (2010, 2020) +DEFAULT_SHAPEFILE_VINTAGE = 2010 +# the vintage `shapefile_urls()` will default to in the 5.0 release +FUTURE_DEFAULT_SHAPEFILE_VINTAGE = 2020 + _lookup_cache: Dict[str, "State"] = {} @@ -58,28 +65,63 @@ def __repr__(self) -> str: def __str__(self) -> str: return self.name - def shapefile_urls(self) -> Optional[Dict[str, str]]: + def shapefile_urls(self, vintage: Optional[int] = None) -> Optional[Dict[str, str]]: """Shapefiles are available directly from the US Census Bureau: https://www.census.gov/cgi-bin/geo/shapefiles/index.php + + `vintage` selects the TIGER/Line vintage, either 2010 or 2020. It + currently defaults to 2010, but in the 5.0 release the default will + change to 2020, so calling without a `vintage` raises a + DeprecationWarning. Pass `vintage=2010` to keep today's URLs, or + `vintage=2020` to opt in to the new default now. + + Note that the Census Bureau stopped publishing per-state files for + some layers after 2010. In the 2020 vintage the "cd", "county", + "state" and "zcta" URLs therefore point at nationwide files, which + are identical for every state. """ + if vintage is None: + warnings.warn( + f"shapefile_urls() currently defaults to the {DEFAULT_SHAPEFILE_VINTAGE} TIGER/Line vintage, " + f"but will default to the {FUTURE_DEFAULT_SHAPEFILE_VINTAGE} vintage in us 5.0. " + f"Pass vintage={DEFAULT_SHAPEFILE_VINTAGE} to keep the current URLs, or " + f"vintage={FUTURE_DEFAULT_SHAPEFILE_VINTAGE} to opt in to the new default now.", + DeprecationWarning, + stacklevel=2, + ) + vintage = DEFAULT_SHAPEFILE_VINTAGE + fips = self.fips if not fips: return None - base = "https://www2.census.gov/geo/tiger/TIGER2010/" - urls = { - "tract": urljoin(base, f"TRACT/2010/tl_2010_{fips}_tract10.zip"), - "cd": urljoin(base, f"CD/111/tl_2010_{fips}_cd111.zip"), - "county": urljoin(base, f"COUNTY/2010/tl_2010_{fips}_county10.zip"), - "state": urljoin(base, f"STATE/2010/tl_2010_{fips}_state10.zip"), - "zcta": urljoin(base, f"ZCTA5/2010/tl_2010_{fips}_zcta510.zip"), - "block": urljoin(base, f"TABBLOCK/2010/tl_2010_{fips}_tabblock10.zip"), - "blockgroup": urljoin(base, f"BG/2010/tl_2010_{fips}_bg10.zip"), - } - - return urls + if vintage == 2010: + base = "https://www2.census.gov/geo/tiger/TIGER2010/" + return { + "tract": urljoin(base, f"TRACT/2010/tl_2010_{fips}_tract10.zip"), + "cd": urljoin(base, f"CD/111/tl_2010_{fips}_cd111.zip"), + "county": urljoin(base, f"COUNTY/2010/tl_2010_{fips}_county10.zip"), + "state": urljoin(base, f"STATE/2010/tl_2010_{fips}_state10.zip"), + "zcta": urljoin(base, f"ZCTA5/2010/tl_2010_{fips}_zcta510.zip"), + "block": urljoin(base, f"TABBLOCK/2010/tl_2010_{fips}_tabblock10.zip"), + "blockgroup": urljoin(base, f"BG/2010/tl_2010_{fips}_bg10.zip"), + } + + if vintage == 2020: + base = "https://www2.census.gov/geo/tiger/TIGER2020/" + return { + "tract": urljoin(base, f"TRACT/tl_2020_{fips}_tract.zip"), + "cd": urljoin(base, "CD/tl_2020_us_cd116.zip"), + "county": urljoin(base, "COUNTY/tl_2020_us_county.zip"), + "state": urljoin(base, "STATE/tl_2020_us_state.zip"), + "zcta": urljoin(base, "ZCTA520/tl_2020_us_zcta520.zip"), + "block": urljoin(base, f"TABBLOCK20/tl_2020_{fips}_tabblock20.zip"), + "blockgroup": urljoin(base, f"BG/tl_2020_{fips}_bg.zip"), + } + + raise ValueError(f"unsupported shapefile vintage {vintage!r}, expected one of {SHAPEFILE_VINTAGES}") def lookup( diff --git a/us/tests/test_us.py b/us/tests/test_us.py index 5dc4196..ca5317e 100644 --- a/us/tests/test_us.py +++ b/us/tests/test_us.py @@ -1,4 +1,10 @@ +import os.path import re +import time +import urllib.error +import urllib.request +import warnings +from concurrent.futures import ThreadPoolExecutor from itertools import chain import jellyfish @@ -262,20 +268,159 @@ def test_cli_lookup(capsys, monkeypatch): # shapefiles +SHAPEFILE_REGIONS = {"block", "blockgroup", "cd", "county", "state", "tract", "zcta"} -@pytest.mark.skip -def test_head(): - # `requests` is intentionally not declared as a dependency; this test - # is permanently skipped and the import is dead code for ty. - import requests # ty: ignore[unresolved-import] +# layers the Census Bureau only publishes as a single nationwide file in the +# 2020 vintage, so the URL is the same no matter which state you ask for +NATIONWIDE_2020_REGIONS = {"cd", "county", "state", "zcta"} +TIGER2020 = "https://www2.census.gov/geo/tiger/TIGER2020/" +TIGER2010 = "https://www2.census.gov/geo/tiger/TIGER2010/" + + +def test_shapefile_urls_default_to_2010(): + # the default is still 2010 until the 5.0 release + assert us.states.DEFAULT_SHAPEFILE_VINTAGE == 2010 + + with pytest.warns(DeprecationWarning): + urls = us.states.MD.shapefile_urls() + + assert urls == { + "tract": TIGER2010 + "TRACT/2010/tl_2010_24_tract10.zip", + "cd": TIGER2010 + "CD/111/tl_2010_24_cd111.zip", + "county": TIGER2010 + "COUNTY/2010/tl_2010_24_county10.zip", + "state": TIGER2010 + "STATE/2010/tl_2010_24_state10.zip", + "zcta": TIGER2010 + "ZCTA5/2010/tl_2010_24_zcta510.zip", + "block": TIGER2010 + "TABBLOCK/2010/tl_2010_24_tabblock10.zip", + "blockgroup": TIGER2010 + "BG/2010/tl_2010_24_bg10.zip", + } + + +def test_shapefile_urls_default_matches_explicit_2010(): + with pytest.warns(DeprecationWarning): + default_urls = us.states.MD.shapefile_urls() + + assert default_urls == us.states.MD.shapefile_urls(vintage=2010) + + +@pytest.mark.parametrize("state", [us.states.MD, us.states.PR]) +def test_shapefile_urls_default_warns_about_the_5_0_change(state): + with pytest.warns(DeprecationWarning) as record: + state.shapefile_urls() + + assert len(record) == 1 + + message = str(record[0].message) + assert "shapefile_urls()" in message + assert "2010" in message + assert "2020" in message + assert "5.0" in message + # the warning should tell callers how to pin either vintage + assert "vintage=2010" in message + assert "vintage=2020" in message + + # stacklevel should point the warning at the caller, not at us/states.py + assert os.path.basename(record[0].filename) == os.path.basename(__file__) + + +def test_shapefile_urls_explicit_vintage_does_not_warn(): + # passing a vintage is how callers opt out of the deprecation warning + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + + for vintage in us.states.SHAPEFILE_VINTAGES: + assert us.states.MD.shapefile_urls(vintage=vintage) is not None + + +def test_shapefile_urls_2020(): + # the vintage that will become the default in 5.0 + assert us.states.FUTURE_DEFAULT_SHAPEFILE_VINTAGE == 2020 + + urls = us.states.MD.shapefile_urls(vintage=2020) + assert urls == { + "tract": TIGER2020 + "TRACT/tl_2020_24_tract.zip", + "cd": TIGER2020 + "CD/tl_2020_us_cd116.zip", + "county": TIGER2020 + "COUNTY/tl_2020_us_county.zip", + "state": TIGER2020 + "STATE/tl_2020_us_state.zip", + "zcta": TIGER2020 + "ZCTA520/tl_2020_us_zcta520.zip", + "block": TIGER2020 + "TABBLOCK20/tl_2020_24_tabblock20.zip", + "blockgroup": TIGER2020 + "BG/tl_2020_24_bg.zip", + } + + +@pytest.mark.parametrize("vintage", us.states.SHAPEFILE_VINTAGES) +def test_shapefile_urls_cover_every_state_and_territory(vintage): + for state in us.STATES_AND_TERRITORIES: + urls = state.shapefile_urls(vintage=vintage) + assert urls is not None, state + assert set(urls) == SHAPEFILE_REGIONS, state + + for region, url in urls.items(): + assert url.startswith(f"https://www2.census.gov/geo/tiger/TIGER{vintage}/"), url + assert url.endswith(".zip"), url + if vintage == 2020 and region in NATIONWIDE_2020_REGIONS: + assert "_us_" in url + else: + assert f"_{state.fips}_" in url, url + + +def test_shapefile_urls_without_fips(): + # obsolete states have no FIPS code, so there is nothing to build a URL from + for state in us.OBSOLETE: + assert state.fips is None + for vintage in us.states.SHAPEFILE_VINTAGES: + assert state.shapefile_urls(vintage=vintage) is None + + +def test_shapefile_urls_unsupported_vintage(): + with pytest.raises(ValueError, match="unsupported shapefile vintage"): + us.states.MD.shapefile_urls(vintage=2000) + + +def _head_status(url: str, attempts: int = 3) -> int: + """HEAD `url` and return its status code.""" + + request = urllib.request.Request(url, method="HEAD") + + for attempt in range(1, attempts + 1): + try: + with urllib.request.urlopen(request, timeout=60) as resp: + return resp.getcode() + except urllib.error.HTTPError as exc: + # a real answer from the server, including a 404 for a bad URL + return exc.code + except urllib.error.URLError: + # DNS and connection hiccups are common when firing off a few + # hundred requests, so retry before calling the URL broken + if attempt == attempts: + raise + time.sleep(attempt) + + raise AssertionError("unreachable") + + +@pytest.mark.network +@pytest.mark.parametrize("vintage", us.states.SHAPEFILE_VINTAGES) +def test_shapefile_urls_are_live(vintage): + """Check every generated URL against the Census Bureau. + + Deselected by default because it needs network access and makes a few + hundred requests. Run it with `uv run pytest -m network`. + """ + urls = set() for state in us.STATES_AND_TERRITORIES: - urls = state.shapefile_urls() - if urls is None: - continue - for url in urls.values(): - resp = requests.head(url) - assert resp.status_code == 200 + state_urls = state.shapefile_urls(vintage=vintage) + assert state_urls is not None + urls.update(state_urls.values()) + + # the nationwide 2020 layers repeat across states, so check each URL once + unique_urls = sorted(urls) + + with ThreadPoolExecutor(max_workers=4) as pool: + statuses = dict(zip(unique_urls, pool.map(_head_status, unique_urls))) + + broken = {url: status for url, status in statuses.items() if status != 200} + assert not broken, f"{len(broken)} of {len(unique_urls)} URLs did not return 200: {broken}" # counts