|
47 | 47 | import mailbox |
48 | 48 | import re |
49 | 49 | import sys |
50 | | -from email.utils import parseaddr |
51 | 50 | from pathlib import Path |
52 | 51 | from typing import Dict, Iterable, List, Tuple |
53 | 52 |
|
@@ -85,15 +84,26 @@ def download_mbox(list_name: str, domain: str, ym: str, out_dir: Path) -> Path: |
85 | 84 | out_path = out_dir / f"{list_name}@{domain}-{ym}.mbox" |
86 | 85 |
|
87 | 86 | headers = {"User-Agent": "apache-list-mentions/1.0"} |
88 | | - r = requests.get(url, headers=headers, timeout=120) |
89 | | - |
90 | | - if r.status_code == 404 or not r.content.strip(): |
| 87 | + with requests.get(url, headers=headers, timeout=120, stream=True) as r: |
| 88 | + if r.status_code == 404: |
| 89 | + raise FileNotFoundError(f"No archive found for {ym}") |
| 90 | + r.raise_for_status() |
| 91 | + |
| 92 | + # Stream to a temp file so an interrupted download is never reused as a cached archive |
| 93 | + tmp_path = out_path.with_suffix(".mbox.part") |
| 94 | + has_content = False |
| 95 | + try: |
| 96 | + with tmp_path.open("wb") as f: |
| 97 | + for chunk in r.iter_content(chunk_size=1024 * 1024): |
| 98 | + has_content = has_content or bool(chunk.strip()) |
| 99 | + f.write(chunk) |
| 100 | + except BaseException: |
| 101 | + tmp_path.unlink(missing_ok=True) |
| 102 | + raise |
| 103 | + |
| 104 | + if not has_content: |
| 105 | + tmp_path.unlink(missing_ok=True) |
91 | 106 | raise FileNotFoundError(f"No archive found for {ym}") |
92 | | - |
93 | | - r.raise_for_status() |
94 | | - # Write via a temp file so an interrupted download is never reused as a cached archive |
95 | | - tmp_path = out_path.with_suffix(".mbox.part") |
96 | | - tmp_path.write_bytes(r.content) |
97 | 107 | tmp_path.replace(out_path) |
98 | 108 | return out_path |
99 | 109 |
|
|
0 commit comments