-
Notifications
You must be signed in to change notification settings - Fork 0
Fix: sync-merge commits misclassified as direct commits #337
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
66cce8a
f46ca68
965ee69
5627697
04d0d27
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -56,6 +56,10 @@ class DataMiner: | |
|
|
||
| def __init__(self, github_instance: Github, rate_limiter: GithubRateLimiter): | ||
| self.github_instance = github_instance | ||
| # dev note: PyGithub paginated results are lazy - the HTTP requests happen while iterating, | ||
| # not on the initial call. Call sites that need a list must materialize it (e.g. via | ||
| # `self._safe_call(lambda: list(x.get_foo()))()`) inside the safe-call, otherwise pagination | ||
| # errors are raised outside of it and go uncaught. | ||
| self._safe_call = safe_call_decorator(rate_limiter) | ||
|
|
||
| def mine_data(self) -> MinedData: | ||
|
|
@@ -74,9 +78,9 @@ def mine_data(self) -> MinedData: | |
| if data.release is not None: | ||
| prefer_published = ActionInputs.get_published_at() | ||
| if prefer_published and getattr(data.release, "published_at", None) is not None: | ||
| data.since = data.release.published_at # type: ignore[assignment] | ||
| data.since = data.release.published_at | ||
| elif getattr(data.release, "created_at", None) is not None: | ||
| data.since = data.release.created_at # type: ignore[assignment] | ||
| data.since = data.release.created_at | ||
| else: | ||
| data.since = None | ||
|
|
||
|
|
@@ -118,7 +122,15 @@ def _handle_compare_mode(self, repo: Repository, data: MinedData) -> None: | |
| to_tag, | ||
| ) | ||
| sys.exit(1) | ||
| compare_commits: list[GithubCommit] = list(comparison.commits) | ||
| compare_commits_result = self._safe_call(lambda: list(comparison.commits))() | ||
| if compare_commits_result is None: | ||
| logger.error( | ||
| "Compare API failed while retrieving commits for '%s'...'%s'. Ending!", | ||
| from_tag, | ||
| to_tag, | ||
| ) | ||
| sys.exit(1) | ||
| compare_commits: list[GithubCommit] = compare_commits_result | ||
| total_commits = getattr(comparison, "total_commits", None) | ||
| if isinstance(total_commits, int) and total_commits > len(compare_commits): | ||
| logger.warning( | ||
|
|
@@ -136,18 +148,32 @@ def _handle_compare_mode(self, repo: Repository, data: MinedData) -> None: | |
| data.commits = {c: data.home_repository for c in compare_commits} | ||
| pr_numbers = self._extract_pr_numbers_from_commits(compare_commits) | ||
| pulls: dict[PullRequest, Repository] = {} | ||
| pr_commit_shas: set[str] = set() | ||
| for number in sorted(pr_numbers): | ||
| pr = self._safe_call(repo.get_pull)(number) | ||
| if pr is not None: | ||
| # Store each PR with its source repository for downstream filtering and processing. | ||
| # In compare mode, all PRs come from home_repository; cross-repo is handled elsewhere. | ||
| pulls[pr] = data.home_repository | ||
| # dev note: pull.get_commits() returns all commits GitHub associates with the PR, | ||
| # including sync-merge commits (base branch merged back into the PR branch) whose | ||
| # messages don't match _PR_NUMBER_RE. Excluding them by SHA (rather than by message | ||
| # pattern) prevents them being misclassified as stand-alone "direct commits". | ||
| # merge_commit_sha is added separately: for a rebase-merge, get_commits() still | ||
| # reports the pre-rebase SHAs, not the new SHA(s) landed on the base branch. | ||
| pr_commits = self._safe_call(lambda p=pr: list(p.get_commits()))() | ||
| if pr_commits is not None: | ||
| pr_commit_shas.update(c.sha for c in pr_commits) | ||
| if pr.merge_commit_sha: | ||
| pr_commit_shas.add(pr.merge_commit_sha) | ||
| data.pull_requests = pulls | ||
|
|
||
| # Only include commits that don't have a PR reference | ||
| # (commits identified by PR are redundant with the PR itself) | ||
| # Only include commits that aren't already accounted for by a PR | ||
| # (commits identified by PR, or belonging to a PR's commit list, are redundant with the PR itself) | ||
| commits_without_pr: dict[GithubCommit, Repository] = {} | ||
| for commit in compare_commits: | ||
| if commit.sha in pr_commit_shas: | ||
| continue | ||
| subject = commit.commit.message.splitlines()[0] if commit.commit.message else "" | ||
| has_pr_ref = bool(_PR_NUMBER_RE.search(subject)) | ||
| if not has_pr_ref: | ||
|
|
@@ -202,14 +228,18 @@ def _handle_since_time_mode(self, repo: Repository, data: MinedData) -> None: | |
| self._get_issues(data) | ||
|
|
||
| # Fetch closed PRs and commits, then reduce them by the latest release since time | ||
| pull_requests = list( | ||
| self._safe_call(repo.get_pulls)(state=PullRequestRecord.PR_STATE_CLOSED, base=repo.default_branch) | ||
| pull_requests = ( | ||
| self._safe_call( | ||
| lambda: list(repo.get_pulls(state=PullRequestRecord.PR_STATE_CLOSED, base=repo.default_branch)) | ||
| )() | ||
| or [] | ||
| ) | ||
|
miroslavpojer marked this conversation as resolved.
|
||
| data.pull_requests = {pr: data.home_repository for pr in pull_requests} | ||
| if data.since: | ||
| commits = list(self._safe_call(repo.get_commits)(since=data.since)) | ||
| since = data.since | ||
| commits = self._safe_call(lambda: list(repo.get_commits(since=since)))() or [] | ||
| else: | ||
| commits = list(self._safe_call(repo.get_commits)()) | ||
| commits = self._safe_call(lambda: list(repo.get_commits()))() or [] | ||
| data.commits = {c: data.home_repository for c in commits} | ||
|
|
||
| def mine_missing_sub_issues(self, data: MinedData) -> tuple[dict[Issue, Repository], dict[str, list[PullRequest]]]: | ||
|
|
@@ -456,7 +486,7 @@ def get_latest_release(self, repository: Repository) -> Optional[GitRelease]: | |
|
|
||
| else: | ||
| logger.info("Getting latest release by semantic ordering (could not be the last one by time).") | ||
| gh_releases: list = list(self._safe_call(repository.get_releases)()) | ||
| gh_releases: list = self._safe_call(lambda: list(repository.get_releases()))() or [] | ||
| rls = self.__get_latest_semantic_release(gh_releases) | ||
|
|
||
| if rls is None: | ||
|
|
@@ -486,7 +516,10 @@ def _get_issues(self, data: MinedData) -> None: | |
| logger.info("Fetching issues from repository...") | ||
|
|
||
| if data.release is None: | ||
| issues = list(self._safe_call(data.home_repository.get_issues)(state=IssueRecord.ISSUE_STATE_ALL)) | ||
| issues = ( | ||
| self._safe_call(lambda: list(data.home_repository.get_issues(state=IssueRecord.ISSUE_STATE_ALL)))() | ||
| or [] | ||
| ) | ||
|
Comment on lines
+519
to
+522
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🩺 Stability & Availability | 🟠 Major | ⚡ Quick win 🧩 Analysis chain🏁 Script executed: #!/bin/bash
set -eu
printf '%s\n' '--- miner.py outline ---'
ast-grep outline release_notes_generator/data/miner.py | sed -n '1,180p'
printf '%s\n' '--- miner.py relevant sections ---'
sed -n '1,110p' release_notes_generator/data/miner.py
sed -n '480,545p' release_notes_generator/data/miner.py
printf '%s\n' '--- safe_call_decorator ---'
cat -n release_notes_generator/utils/decorators.py | sed -n '40,90p'
printf '%s\n' '--- related symbols and tests ---'
rg -n -C 3 'issues_since|open_issues|ISSUE_STATE_ALL|safe_call' release_notes_generator tests 2>/dev/null | sed -n '1,240p'Repository: AbsaOSS/generate-release-notes Length of output: 30876 🏁 Script executed: #!/bin/bash
set -eu
python3 - <<'PY'
import ast
from pathlib import Path
miner_path = Path("release_notes_generator/data/miner.py")
decorator_path = Path("release_notes_generator/utils/decorators.py")
miner = ast.parse(miner_path.read_text())
decorators = ast.parse(decorator_path.read_text())
# Confirm the release branch passes the lazy API result through safe_call,
# then materializes it outside the wrapper.
get_issues_calls = []
for node in ast.walk(miner):
if isinstance(node, ast.Assign) and any(
isinstance(target, ast.Name) and target.id in {"issues_since", "open_issues"}
for target in node.targets
):
get_issues_calls.append(
(node.lineno, ast.unparse(node.value))
)
print("release-branch assignments:")
for line, expression in sorted(get_issues_calls):
print(f"{line}: {expression}")
list_assignments = []
for node in ast.walk(miner):
if isinstance(node, ast.Assign) and any(
isinstance(target, ast.Name) and target.id in {"issues_since", "open_issues"}
for target in node.targets
):
if isinstance(node.value, ast.Call) and getattr(node.value.func, "id", None) == "list":
list_assignments.append(node.lineno)
print("materialization lines:", sorted(list_assignments))
# Isolated behavior probe: the wrapper catches errors raised by the wrapped
# call, but not errors raised later while consuming its lazy return value.
class PaginationError(Exception):
pass
def safe_call(method):
def wrapped(*args, **kwargs):
try:
return method(*args, **kwargs)
except Exception:
return None
return wrapped
def lazy_pages():
def iterator():
yield "page-1"
raise PaginationError("pagination failed")
return iterator()
wrapped_lazy_call = safe_call(lambda: lazy_pages())
try:
list(wrapped_lazy_call())
except PaginationError:
print("lazy result + list outside wrapper: pagination error escapes")
wrapped_materialization = safe_call(lambda: list(lazy_pages()))
if wrapped_materialization() is None:
print("list inside wrapper: pagination error is converted to None")
# Confirm safe_call_decorator has a broad exception handler around only method().
factory = next(
node for node in decorators.body
if isinstance(node, ast.FunctionDef) and node.name == "safe_call_decorator"
)
handler_types = []
for node in ast.walk(factory):
if isinstance(node, ast.ExceptHandler):
handler_types.append(ast.unparse(node.type) if node.type else "bare")
print("safe_call exception handlers:", sorted(set(handler_types)))
PYRepository: AbsaOSS/generate-release-notes Length of output: 497 🏁 Script executed: #!/bin/bash
set -eu
python3 - <<'PY'
import ast
from pathlib import Path
miner = ast.parse(Path("release_notes_generator/data/miner.py").read_text())
decorators = ast.parse(Path("release_notes_generator/utils/decorators.py").read_text())
for node in ast.walk(miner):
if isinstance(node, ast.Assign) and any(
isinstance(target, ast.Name)
and target.id in {"issues_since", "open_issues"}
for target in node.targets
):
print(node.lineno, ast.unparse(node.value))
class PaginationError(Exception):
pass
def safe_call(method):
def wrapped(*args, **kwargs):
try:
return method(*args, **kwargs)
except Exception:
return None
return wrapped
def lazy_pages():
def iterator():
yield "page-1"
raise PaginationError
return iterator()
try:
list(safe_call(lambda: lazy_pages())())
except PaginationError:
print("pagination escapes when list() is outside wrapper")
if safe_call(lambda: list(lazy_pages()))() is None:
print("pagination is caught when list() is inside wrapper")
PYRepository: AbsaOSS/generate-release-notes Length of output: 497 🏁 Script executed: #!/bin/bash
set -eu
python3 - <<'PY'
from pathlib import Path
lines = Path("release_notes_generator/data/miner.py").read_text().splitlines()
window = "\n".join(f"{i + 1}: {lines[i]}" for i in range(536, 546))
print(window)
assert "issues_since = self._safe_call(data.home_repository.get_issues)(" in window
assert "open_issues = self._safe_call(data.home_repository.get_issues)(" in window
assert "issues_since = list(issues_since or [])" in window or \
"issues_since = list(issues_since or [])" in "\n".join(lines[544:546])
assert "open_issues = list(open_issues or [])" in "\n".join(lines[544:546])
print("release-mode list() calls are outside the safe-call invocation")
class PaginationError(Exception):
pass
def safe_call(method):
def wrapped(*args, **kwargs):
try:
return method(*args, **kwargs)
except Exception:
return None
return wrapped
def lazy_pages():
def iterator():
yield "page-1"
raise PaginationError("pagination failed")
return iterator()
try:
list(safe_call(lambda: lazy_pages())())
except PaginationError:
print("pagination escapes when list() is outside the wrapper")
if safe_call(lambda: list(lazy_pages()))() is None:
print("pagination is caught when list() is inside the wrapper")
PYRepository: AbsaOSS/generate-release-notes Length of output: 775 🏁 Script executed: #!/bin/bash
set -eu
python3 - <<'PY'
from pathlib import Path
lines = Path("release_notes_generator/data/miner.py").read_text().splitlines()
for i in range(536, 546):
print(f"{i + 1}: {lines[i]}")
class PaginationError(Exception):
pass
def safe_call(method):
def wrapped(*args, **kwargs):
try:
return method(*args, **kwargs)
except Exception:
return None
return wrapped
def lazy_pages():
def iterator():
yield "page-1"
raise PaginationError("pagination failed")
return iterator()
try:
list(safe_call(lambda: lazy_pages())())
except PaginationError:
print("pagination escapes when list() is outside the wrapper")
if safe_call(lambda: list(lazy_pages()))() is None:
print("pagination is caught when list() is inside the wrapper")
PYRepository: AbsaOSS/generate-release-notes Length of output: 712 Materialize release-mode issue pages inside the safe-call wrapper. PyGithub performs pagination during iteration. In release mode, 🤖 Prompt for AI Agents |
||
| data.issues = {i: data.home_repository for i in issues} | ||
|
|
||
| logger.info("Fetched %d issues", len(data.issues.items())) | ||
|
|
@@ -497,9 +530,9 @@ def _get_issues(self, data: MinedData) -> None: | |
| # Ensure data.since is only set if a valid datetime is available | ||
| data.since = None | ||
| if prefer_published and getattr(data.release, "published_at", None) is not None: | ||
| data.since = data.release.published_at # type: ignore[assignment] | ||
| data.since = data.release.published_at | ||
| elif getattr(data.release, "created_at", None) is not None: | ||
| data.since = data.release.created_at # type: ignore[assignment] | ||
| data.since = data.release.created_at | ||
|
|
||
| issues_since = self._safe_call(data.home_repository.get_issues)( | ||
| state=IssueRecord.ISSUE_STATE_ALL, | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🗄️ Data Integrity & Integration | 🟠 Major | ⚡ Quick win
Propagate collection retrieval failures instead of treating them as empty results. If
get_commits()or another collection fetch fails or returnsNone, stop processing or preserve an explicit incomplete state. Otherwise PR commits can be emitted as direct commits, and failed issue or commit retrieval can silently produce incomplete or overly broad release notes. Apply this consistently at the affected collection-retrieval sites.📍 Affects 1 file
release_notes_generator/data/miner.py#L164-L168(this comment)release_notes_generator/data/miner.py#L164-L168🤖 Prompt for AI Agents