Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
# Changelog

## Unreleased

* add a `vintage` argument to `shapefile_urls()`, so URLs can be generated for either the 2010 or the 2020 TIGER/Line vintage. In the 2020 vintage the Census Bureau only publishes nationwide files for the `cd`, `county`, `state` and `zcta` layers, so those URLs are the same for every state.
* `vintage` still defaults to 2010, but that default will change to 2020 in the 5.0 release. Calling `shapefile_urls()` without a `vintage` now raises a `DeprecationWarning`; pass `vintage=2010` to keep the current URLs, or `vintage=2020` to opt in to the new default now.
* add `network`-marked tests that check the generated shapefile URLs against the Census Bureau, deselected by default


## 4.0.0

* add counties, thanks to [Ray Kiddy](https://github.com/rkiddy)
Expand Down
29 changes: 27 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -200,10 +200,10 @@ no fallback at all).

### Shapefiles

You want shapefiles too? As long as you want 2010 shapefiles, we've gotcha covered.
You want shapefiles too? We've gotcha covered.

```
>>> urls = us.states.MD.shapefile_urls()
>>> urls = us.states.MD.shapefile_urls(vintage=2010)
>>> sorted(urls.keys())
['block', 'blockgroup', 'cd', 'county', 'state', 'tract', 'zcta']
>>> urls['block']
Expand All @@ -221,6 +221,23 @@ the following regions:
* state
* zcta

The `vintage` argument selects the TIGER/Line vintage, either 2010 or 2020:

```
>>> us.states.MD.shapefile_urls(vintage=2020)['block']
'https://www2.census.gov/geo/tiger/TIGER2020/TABBLOCK20/tl_2020_24_tabblock20.zip'
```

**Deprecation:** `vintage` currently defaults to 2010, but in us 5.0 the
default will change to 2020. Calling `shapefile_urls()` without a `vintage`
raises a `DeprecationWarning` until then. Pass `vintage=2010` to keep today's
URLs, or `vintage=2020` to opt in to the new default now.

Note that the Census Bureau stopped publishing per-state files for some layers
after 2010. In the 2020 vintage the `cd`, `county`, `state` and `zcta` URLs
point at nationwide files, so they're identical for every state. The `block`
URLs are for 2020 census blocks (`TABBLOCK20`), and `cd` is the 116th Congress.


### Mappings

Expand Down Expand Up @@ -327,6 +344,14 @@ uv sync
uv run pytest
```

Tests marked `network` check the generated shapefile URLs against the Census
Bureau. They're deselected by default because they need network access and
make a few hundred requests. To run them:

```
uv run pytest -m network
```


## Changelog

Expand Down
6 changes: 6 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -40,3 +40,9 @@ dev = [
[tool.ruff]
line-length = 120
target-version = "py38"

[tool.pytest.ini_options]
markers = [
"network: makes live requests to the Census Bureau (deselected by default, run with `-m network`)",
]
addopts = "-m 'not network'"
4 changes: 3 additions & 1 deletion us/cli/states.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,9 @@ def main():

sys.stdout.write("\n")
sys.stdout.write(" shapefiles:\n")
urls = state.shapefile_urls()
# pass the default explicitly: the CLI tracks whatever the library
# default is without tripping its deprecation warning
urls = state.shapefile_urls(vintage=us.states.DEFAULT_SHAPEFILE_VINTAGE)
if urls is not None:
for region, url in urls.items():
sys.stdout.write(" %s: %s\n" % (region, url))
Expand Down
68 changes: 55 additions & 13 deletions us/states.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import os
import re
import warnings
from enum import Enum
from typing import Any, Callable, Dict, Iterable, List, Optional, Type
from urllib.parse import urljoin
Expand All @@ -11,6 +12,12 @@

DC_STATEHOOD = bool(os.environ.get("DC_STATEHOOD"))

# TIGER/Line vintages that `State.shapefile_urls()` knows how to build URLs for.
SHAPEFILE_VINTAGES = (2010, 2020)
DEFAULT_SHAPEFILE_VINTAGE = 2010
# the vintage `shapefile_urls()` will default to in the 5.0 release
FUTURE_DEFAULT_SHAPEFILE_VINTAGE = 2020


_lookup_cache: Dict[str, "State"] = {}

Expand Down Expand Up @@ -58,28 +65,63 @@ def __repr__(self) -> str:
def __str__(self) -> str:
return self.name

def shapefile_urls(self) -> Optional[Dict[str, str]]:
def shapefile_urls(self, vintage: Optional[int] = None) -> Optional[Dict[str, str]]:
"""Shapefiles are available directly from the US Census Bureau:
https://www.census.gov/cgi-bin/geo/shapefiles/index.php

`vintage` selects the TIGER/Line vintage, either 2010 or 2020. It
currently defaults to 2010, but in the 5.0 release the default will
change to 2020, so calling without a `vintage` raises a
DeprecationWarning. Pass `vintage=2010` to keep today's URLs, or
`vintage=2020` to opt in to the new default now.

Note that the Census Bureau stopped publishing per-state files for
some layers after 2010. In the 2020 vintage the "cd", "county",
"state" and "zcta" URLs therefore point at nationwide files, which
are identical for every state.
"""

if vintage is None:
warnings.warn(
f"shapefile_urls() currently defaults to the {DEFAULT_SHAPEFILE_VINTAGE} TIGER/Line vintage, "
f"but will default to the {FUTURE_DEFAULT_SHAPEFILE_VINTAGE} vintage in us 5.0. "
f"Pass vintage={DEFAULT_SHAPEFILE_VINTAGE} to keep the current URLs, or "
f"vintage={FUTURE_DEFAULT_SHAPEFILE_VINTAGE} to opt in to the new default now.",
DeprecationWarning,
stacklevel=2,
)
vintage = DEFAULT_SHAPEFILE_VINTAGE

fips = self.fips

if not fips:
return None

base = "https://www2.census.gov/geo/tiger/TIGER2010/"
urls = {
"tract": urljoin(base, f"TRACT/2010/tl_2010_{fips}_tract10.zip"),
"cd": urljoin(base, f"CD/111/tl_2010_{fips}_cd111.zip"),
"county": urljoin(base, f"COUNTY/2010/tl_2010_{fips}_county10.zip"),
"state": urljoin(base, f"STATE/2010/tl_2010_{fips}_state10.zip"),
"zcta": urljoin(base, f"ZCTA5/2010/tl_2010_{fips}_zcta510.zip"),
"block": urljoin(base, f"TABBLOCK/2010/tl_2010_{fips}_tabblock10.zip"),
"blockgroup": urljoin(base, f"BG/2010/tl_2010_{fips}_bg10.zip"),
}

return urls
if vintage == 2010:
base = "https://www2.census.gov/geo/tiger/TIGER2010/"
return {
"tract": urljoin(base, f"TRACT/2010/tl_2010_{fips}_tract10.zip"),
"cd": urljoin(base, f"CD/111/tl_2010_{fips}_cd111.zip"),
"county": urljoin(base, f"COUNTY/2010/tl_2010_{fips}_county10.zip"),
"state": urljoin(base, f"STATE/2010/tl_2010_{fips}_state10.zip"),
"zcta": urljoin(base, f"ZCTA5/2010/tl_2010_{fips}_zcta510.zip"),
"block": urljoin(base, f"TABBLOCK/2010/tl_2010_{fips}_tabblock10.zip"),
"blockgroup": urljoin(base, f"BG/2010/tl_2010_{fips}_bg10.zip"),
}

if vintage == 2020:
base = "https://www2.census.gov/geo/tiger/TIGER2020/"
return {
"tract": urljoin(base, f"TRACT/tl_2020_{fips}_tract.zip"),
"cd": urljoin(base, "CD/tl_2020_us_cd116.zip"),
"county": urljoin(base, "COUNTY/tl_2020_us_county.zip"),
"state": urljoin(base, "STATE/tl_2020_us_state.zip"),
"zcta": urljoin(base, "ZCTA520/tl_2020_us_zcta520.zip"),
"block": urljoin(base, f"TABBLOCK20/tl_2020_{fips}_tabblock20.zip"),
"blockgroup": urljoin(base, f"BG/tl_2020_{fips}_bg.zip"),
}

raise ValueError(f"unsupported shapefile vintage {vintage!r}, expected one of {SHAPEFILE_VINTAGES}")


def lookup(
Expand Down
167 changes: 156 additions & 11 deletions us/tests/test_us.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
import os.path
import re
import time
import urllib.error
import urllib.request
import warnings
from concurrent.futures import ThreadPoolExecutor
from itertools import chain

import jellyfish
Expand Down Expand Up @@ -262,20 +268,159 @@ def test_cli_lookup(capsys, monkeypatch):

# shapefiles

SHAPEFILE_REGIONS = {"block", "blockgroup", "cd", "county", "state", "tract", "zcta"}

@pytest.mark.skip
def test_head():
# `requests` is intentionally not declared as a dependency; this test
# is permanently skipped and the import is dead code for ty.
import requests # ty: ignore[unresolved-import]
# layers the Census Bureau only publishes as a single nationwide file in the
# 2020 vintage, so the URL is the same no matter which state you ask for
NATIONWIDE_2020_REGIONS = {"cd", "county", "state", "zcta"}

TIGER2020 = "https://www2.census.gov/geo/tiger/TIGER2020/"
TIGER2010 = "https://www2.census.gov/geo/tiger/TIGER2010/"


def test_shapefile_urls_default_to_2010():
# the default is still 2010 until the 5.0 release
assert us.states.DEFAULT_SHAPEFILE_VINTAGE == 2010

with pytest.warns(DeprecationWarning):
urls = us.states.MD.shapefile_urls()

assert urls == {
"tract": TIGER2010 + "TRACT/2010/tl_2010_24_tract10.zip",
"cd": TIGER2010 + "CD/111/tl_2010_24_cd111.zip",
"county": TIGER2010 + "COUNTY/2010/tl_2010_24_county10.zip",
"state": TIGER2010 + "STATE/2010/tl_2010_24_state10.zip",
"zcta": TIGER2010 + "ZCTA5/2010/tl_2010_24_zcta510.zip",
"block": TIGER2010 + "TABBLOCK/2010/tl_2010_24_tabblock10.zip",
"blockgroup": TIGER2010 + "BG/2010/tl_2010_24_bg10.zip",
}


def test_shapefile_urls_default_matches_explicit_2010():
with pytest.warns(DeprecationWarning):
default_urls = us.states.MD.shapefile_urls()

assert default_urls == us.states.MD.shapefile_urls(vintage=2010)


@pytest.mark.parametrize("state", [us.states.MD, us.states.PR])
def test_shapefile_urls_default_warns_about_the_5_0_change(state):
with pytest.warns(DeprecationWarning) as record:
state.shapefile_urls()

assert len(record) == 1

message = str(record[0].message)
assert "shapefile_urls()" in message
assert "2010" in message
assert "2020" in message
assert "5.0" in message
# the warning should tell callers how to pin either vintage
assert "vintage=2010" in message
assert "vintage=2020" in message

# stacklevel should point the warning at the caller, not at us/states.py
assert os.path.basename(record[0].filename) == os.path.basename(__file__)


def test_shapefile_urls_explicit_vintage_does_not_warn():
# passing a vintage is how callers opt out of the deprecation warning
with warnings.catch_warnings():
warnings.simplefilter("error", DeprecationWarning)

for vintage in us.states.SHAPEFILE_VINTAGES:
assert us.states.MD.shapefile_urls(vintage=vintage) is not None


def test_shapefile_urls_2020():
# the vintage that will become the default in 5.0
assert us.states.FUTURE_DEFAULT_SHAPEFILE_VINTAGE == 2020

urls = us.states.MD.shapefile_urls(vintage=2020)
assert urls == {
"tract": TIGER2020 + "TRACT/tl_2020_24_tract.zip",
"cd": TIGER2020 + "CD/tl_2020_us_cd116.zip",
"county": TIGER2020 + "COUNTY/tl_2020_us_county.zip",
"state": TIGER2020 + "STATE/tl_2020_us_state.zip",
"zcta": TIGER2020 + "ZCTA520/tl_2020_us_zcta520.zip",
"block": TIGER2020 + "TABBLOCK20/tl_2020_24_tabblock20.zip",
"blockgroup": TIGER2020 + "BG/tl_2020_24_bg.zip",
}


@pytest.mark.parametrize("vintage", us.states.SHAPEFILE_VINTAGES)
def test_shapefile_urls_cover_every_state_and_territory(vintage):
for state in us.STATES_AND_TERRITORIES:
urls = state.shapefile_urls(vintage=vintage)
assert urls is not None, state
assert set(urls) == SHAPEFILE_REGIONS, state

for region, url in urls.items():
assert url.startswith(f"https://www2.census.gov/geo/tiger/TIGER{vintage}/"), url
assert url.endswith(".zip"), url
if vintage == 2020 and region in NATIONWIDE_2020_REGIONS:
assert "_us_" in url
else:
assert f"_{state.fips}_" in url, url


def test_shapefile_urls_without_fips():
# obsolete states have no FIPS code, so there is nothing to build a URL from
for state in us.OBSOLETE:
assert state.fips is None
for vintage in us.states.SHAPEFILE_VINTAGES:
assert state.shapefile_urls(vintage=vintage) is None


def test_shapefile_urls_unsupported_vintage():
with pytest.raises(ValueError, match="unsupported shapefile vintage"):
us.states.MD.shapefile_urls(vintage=2000)


def _head_status(url: str, attempts: int = 3) -> int:
"""HEAD `url` and return its status code."""

request = urllib.request.Request(url, method="HEAD")

for attempt in range(1, attempts + 1):
try:
with urllib.request.urlopen(request, timeout=60) as resp:
return resp.getcode()
except urllib.error.HTTPError as exc:
# a real answer from the server, including a 404 for a bad URL
return exc.code
except urllib.error.URLError:
# DNS and connection hiccups are common when firing off a few
# hundred requests, so retry before calling the URL broken
if attempt == attempts:
raise
time.sleep(attempt)

raise AssertionError("unreachable")


@pytest.mark.network
@pytest.mark.parametrize("vintage", us.states.SHAPEFILE_VINTAGES)
def test_shapefile_urls_are_live(vintage):
"""Check every generated URL against the Census Bureau.

Deselected by default because it needs network access and makes a few
hundred requests. Run it with `uv run pytest -m network`.
"""
urls = set()
for state in us.STATES_AND_TERRITORIES:
urls = state.shapefile_urls()
if urls is None:
continue
for url in urls.values():
resp = requests.head(url)
assert resp.status_code == 200
state_urls = state.shapefile_urls(vintage=vintage)
assert state_urls is not None
urls.update(state_urls.values())

# the nationwide 2020 layers repeat across states, so check each URL once
unique_urls = sorted(urls)

with ThreadPoolExecutor(max_workers=4) as pool:
statuses = dict(zip(unique_urls, pool.map(_head_status, unique_urls)))

broken = {url: status for url, status in statuses.items() if status != 200}
assert not broken, f"{len(broken)} of {len(unique_urls)} URLs did not return 200: {broken}"


# counts
Expand Down