diff --git a/src/linkml_reference_validator/etl/sources/pmid.py b/src/linkml_reference_validator/etl/sources/pmid.py index 097b376..34ec16c 100644 --- a/src/linkml_reference_validator/etl/sources/pmid.py +++ b/src/linkml_reference_validator/etl/sources/pmid.py @@ -17,7 +17,6 @@ from Bio import Entrez # type: ignore from bs4 import BeautifulSoup # type: ignore -import requests # type: ignore from linkml_reference_validator.models import ReferenceContent, ReferenceValidationConfig from linkml_reference_validator.etl.sources.base import ReferenceSource, ReferenceSourceRegistry @@ -335,138 +334,3 @@ def _parse_publication_types( ] return types if types else None - - def _fetch_pmc_fulltext( - self, pmid: str, config: ReferenceValidationConfig - ) -> tuple[Optional[str], str]: - """Attempt to fetch full text from PMC. - - Args: - pmid: PubMed ID - config: Configuration for rate limiting - - Returns: - Tuple of (full_text, content_type) - """ - pmcid = self._get_pmcid(pmid, config) - if not pmcid: - return None, "no_pmc" - - full_text = self._fetch_pmc_xml(pmcid, config) - if full_text and len(full_text) > 1000: - return full_text, "full_text_xml" - - full_text = self._fetch_pmc_html(pmcid, config) - if full_text and len(full_text) > 1000: - return full_text, "full_text_html" - - return None, "pmc_restricted" - - def _get_pmcid(self, pmid: str, config: ReferenceValidationConfig) -> Optional[str]: - """Get PMC ID for a PubMed ID. - - Args: - pmid: PubMed ID - config: Configuration for rate limiting - - Returns: - PMC ID if available - """ - time.sleep(config.rate_limit_delay) - - try: - handle = Entrez.elink( - dbfrom="pubmed", db="pmc", id=pmid, linkname="pubmed_pmc" - ) - except Exception as exc: - logger.warning("Failed to link PMID:%s to PMC: %s", pmid, exc) - return None - - try: - result = Entrez.read(handle) - except Exception as exc: - logger.warning( - "Failed to read PMC link for PMID:%s: %s", pmid, exc) - return None - finally: - handle.close() - - if isinstance(result, list) and result and isinstance(result[0], dict): - link_set_db = result[0].get("LinkSetDb", []) - if isinstance(link_set_db, list) and link_set_db: - links = link_set_db[0].get("Link", []) - if isinstance(links, list) and links: - first_link = links[0] - if isinstance(first_link, dict) and "Id" in first_link: - return str(first_link["Id"]) - - return None - - def _fetch_pmc_xml( - self, pmcid: str, config: ReferenceValidationConfig - ) -> Optional[str]: - """Fetch full text from PMC XML API. - - Args: - pmcid: PMC ID - config: Configuration for rate limiting - - Returns: - Extracted text from XML - """ - time.sleep(config.rate_limit_delay) - - handle = Entrez.efetch( - db="pmc", id=pmcid, rettype="xml", retmode="xml") - xml_content = handle.read() - handle.close() - - if isinstance(xml_content, bytes): - xml_content = xml_content.decode("utf-8") - - if "cannot be obtained" in xml_content.lower() or "restricted" in xml_content.lower(): - return None - - soup = BeautifulSoup(xml_content, "xml") - body = soup.find("body") - - if body: - paragraphs = body.find_all("p") - if paragraphs: - text = "\n\n".join(p.get_text() for p in paragraphs) - return text - - return None - - def _fetch_pmc_html( - self, pmcid: str, config: ReferenceValidationConfig - ) -> Optional[str]: - """Fetch full text from PMC HTML as fallback. - - Args: - pmcid: PMC ID - config: Configuration for rate limiting - - Returns: - Extracted text from HTML - """ - time.sleep(config.rate_limit_delay) - - url = f"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC{pmcid}/" - - response = requests.get(url, timeout=30) - if response.status_code != 200: - return None - - soup = BeautifulSoup(response.content, "html.parser") - article_body = soup.find("div", class_="article-body") or soup.find( - "div", class_="tsec" - ) - - if article_body: - paragraphs = article_body.find_all("p") - if paragraphs: - text = "\n\n".join(p.get_text() for p in paragraphs) - return text - - return None diff --git a/src/linkml_reference_validator/plugins/REVIEW.md b/src/linkml_reference_validator/plugins/REVIEW.md index 38f62c4..d15f0d9 100644 --- a/src/linkml_reference_validator/plugins/REVIEW.md +++ b/src/linkml_reference_validator/plugins/REVIEW.md @@ -229,7 +229,7 @@ Comprehensive Repository Review Report 2. CLI Not Tested: Violates: "Always include CLI tests" 3. Insufficient Doctests for Complex Functions: - Some complex functions lack doctests (e.g., _fetch_pmc_fulltext) + Some complex functions lack doctests Architecture Strengths @@ -247,13 +247,8 @@ Comprehensive Repository Review Report reference_fetcher.py (68% coverage) Untested lines (54 total): - - Lines 224-232: _fetch_pmc_fulltext logic - - Lines 265-286: _fetch_pmc_xml parsing - - Lines 297-314: _fetch_pmc_html scraping - Lines 139-140, 170-172: Error handling - Impact: PMC full-text retrieval completely untested - reference_validation_plugin.py (56% coverage) Untested lines (55 total): diff --git a/tests/test_fulltext_providers.py b/tests/test_fulltext_providers.py index d8e8368..eddabf0 100644 --- a/tests/test_fulltext_providers.py +++ b/tests/test_fulltext_providers.py @@ -173,3 +173,14 @@ def test_locate_returns_text_from_xml(self, config): assert loc.format_hint == "xml" assert loc.provider == "pmc" assert "Sentence 0 of the body." in loc.text + + @patch("linkml_reference_validator.etl.fulltext.pmc.Entrez.elink") + def test_resolve_pmcid_handles_entrez_error(self, mock_elink, config): + """Should return None when the Entrez elink call fails.""" + from linkml_reference_validator.etl.fulltext.pmc import PMCFullTextProvider + + mock_elink.side_effect = RuntimeError( + "Couldn't resolve #exLinkSrv2, the address table is empty." + ) + + assert PMCFullTextProvider()._resolve_pmcid("12112053", config) is None diff --git a/tests/test_sources.py b/tests/test_sources.py index 2a50d27..4f66f61 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -297,27 +297,6 @@ def test_can_handle_pmid(self, source): assert source.can_handle("PMID 12345678") assert not source.can_handle("DOI:10.1234/test") - @patch("linkml_reference_validator.etl.sources.pmid.Entrez.read") - @patch("linkml_reference_validator.etl.sources.pmid.Entrez.elink") - def test_get_pmcid_handles_entrez_error( - self, - mock_elink, - mock_read, - source, - config, - ): - """Should return None when Entrez.read raises an error.""" - handle = MagicMock() - mock_elink.return_value = handle - mock_read.side_effect = RuntimeError( - "Couldn't resolve #exLinkSrv2, the address table is empty." - ) - - result = source._get_pmcid("12112053", config) - - assert result is None - handle.close.assert_called_once() - # --- Publication types (issue #56) ----------------------------------- _ARTICLE_XML = """