ci: publish wheels as immutable GitHub releases - #834
Open
luhenry wants to merge 3 commits into
Open
Conversation
luhenry
marked this pull request as ready for review
September 2, 2026 12:07
luhenry
marked this pull request as draft
September 2, 2026 12:07
Member
Author
|
I'm writing and testing the scripts that will download all previous releases to GitLab and publish them as GitHub releases |
Replace GitLab publishing with a reusable release workflow and generate the Simple API from package YAML metadata. Signed-off-by: Ludovic Henry <git@ludovic.dev>
luhenry
marked this pull request as ready for review
September 2, 2026 14:26
The script that was used to migrate it all is the following:
```
"""One-off GitLab PyPI registry -> immutable GitHub Releases migration.
Dry-run by default. Pass --execute to create releases and update docs/packages/*.yaml.
Requires: requests-cache, pyyaml, and an authenticated `gh` CLI with contents write.
The target repository must have immutable releases enabled.
"""
import argparse
import hashlib
import json
import os
import re
import subprocess
import sys
import tempfile
import time
from collections import defaultdict
from datetime import datetime, timezone
from email.parser import Parser
from html.parser import HTMLParser
from pathlib import Path
from urllib.parse import urljoin, urlparse
import zipfile
import requests_cache
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
import yaml
GITLAB_PROJECT_ID = "56254198"
GITLAB_SIMPLE = (
f"https://gitlab.com/api/v4/projects/{GITLAB_PROJECT_ID}/packages/pypi/simple/"
)
GITLAB_PACKAGES = f"https://gitlab.com/api/v4/projects/{GITLAB_PROJECT_ID}/packages"
GITLAB_PACKAGE_UI = "https://gitlab.com/riseproject/python/wheel_builder/-/packages"
REPO = "riseproject-dev/python-wheels"
DOCS = Path("docs/packages")
STATE = Path(".git/pw-scratch/release-migration/state.json")
REPORT = Path(".git/pw-scratch/release-migration/report.json")
def normalize(name):
return re.sub(r"[-_.]+", "-", name).lower()
class Links(HTMLParser):
def __init__(self):
super().__init__()
self.links = []
self.current = None
self.text = []
def handle_starttag(self, tag, attrs):
if tag == "a":
self.current = dict(attrs)
self.text = []
def handle_data(self, data):
if self.current is not None:
self.text.append(data)
def handle_endtag(self, tag):
if tag == "a" and self.current is not None:
self.links.append((self.current, "".join(self.text).strip()))
self.current = None
self.text = []
def get_links(session, url):
response = session.get(url, timeout=(30, 300))
response.raise_for_status()
parser = Links()
parser.feed(response.text)
return parser.links
def get_package_ids(session):
"""Return the newest GitLab package ID for each normalized name/version."""
package_ids = {}
page = 1
while True:
response = session.get(
GITLAB_PACKAGES,
params={"package_type": "pypi", "per_page": 100, "page": page},
timeout=(30, 300),
)
response.raise_for_status()
packages = response.json()
for package in packages:
key = (normalize(package["name"]), str(package["version"]))
package_ids[key] = max(package["id"], package_ids.get(key, 0))
next_page = response.headers.get("X-Next-Page")
if not next_page:
break
page = int(next_page)
return package_ids
def run(*args, capture=False):
return subprocess.run(args, check=True, text=True, capture_output=capture)
def wheel_requires_python(path):
with zipfile.ZipFile(path) as archive:
metadata_path = next(
n for n in archive.namelist() if n.endswith(".dist-info/METADATA")
)
return (
Parser()
.parsestr(archive.read(metadata_path).decode())
.get("Requires-Python")
)
def load_state():
if STATE.exists():
return json.loads(STATE.read_text())
return {
"timestamp": datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S"),
"completed": {},
}
def save_state(state):
STATE.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n")
def existing_gpl_sources(version_item):
comment = version_item.get("comment")
if not comment or "gpl-sources.tar" not in comment:
return None
match = re.search(r"(https://github\.com/[^>`]+/gpl-sources\.tar)", comment)
if not match:
return None
description = re.search(r"bundled GPL libraries(?: \(([^)]+)\))?", comment)
result = {"url": match.group(1), "filename": "gpl-sources.tar"}
if description and description.group(1):
result["description"] = description.group(1)
return result
def add_yaml_release(package, version, release):
path = DOCS / f"{package}.yaml"
if not path.exists():
raise RuntimeError(f"missing package YAML: {path}")
data = yaml.safe_load(path.read_text())
item = next(
(v for v in data.get("versions", []) if str(v.get("version")) == version),
None,
)
if item is None:
item = {"version": version}
data.setdefault("versions", []).append(item)
if release.get("gpl-sources"):
item.pop("comment", None)
releases = item.setdefault("releases", [])
if not any(r.get("tag") == release["tag"] for r in releases):
releases.append(release)
path.write_text(yaml.safe_dump(data, sort_keys=False, allow_unicode=True))
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--execute", action="store_true")
args = parser.parse_args()
state = load_state()
session = requests_cache.CachedSession(
cache_name="/tmp/cache/gitlab-package-migration",
backend="filesystem",
expire_after=-1,
)
gitlab_token = os.environ.get("GITLAB_TOKEN")
if gitlab_token:
session.headers.update({"PRIVATE-TOKEN": gitlab_token})
retries = Retry(
total=8, backoff_factor=2, status_forcelist=(429, 500, 502, 503, 504)
)
session.mount("https://", HTTPAdapter(max_retries=retries))
package_ids = get_package_ids(session)
root_links = get_links(session, GITLAB_SIMPLE)
package_urls = {}
for attrs, text in root_links:
href = attrs.get("href")
if href and text:
package_urls[normalize(text)] = urljoin(GITLAB_SIMPLE, href)
groups = defaultdict(list)
for number, (package, package_url) in enumerate(sorted(package_urls.items()), 1):
print(f"[{number}/{len(package_urls)}] indexing {package}")
for attrs, filename in get_links(session, package_url):
if not filename.endswith(".whl"):
continue
href = urljoin(package_url, attrs["href"])
groups[package].append((href, filename, attrs.get("data-requires-python")))
report = {
"packages": len(package_urls),
"package_pages": len(groups),
"files": sum(map(len, groups.values())),
"completed": [],
}
print(json.dumps(report, indent=2))
if args.execute:
immutable = json.loads(
run("gh", "api", f"repos/{REPO}/immutable-releases", capture=True).stdout
)
if not immutable.get("enabled"):
raise RuntimeError("immutable releases are not enabled")
yaml_packages = {
normalize(yaml.safe_load(path.read_text())["package-name"])
for path in DOCS.glob("*.yaml")
}
missing_from_gitlab = sorted(yaml_packages - package_urls.keys())
if missing_from_gitlab:
raise RuntimeError(
"YAML packages absent from GitLab simple index: "
+ ", ".join(missing_from_gitlab)
)
report["yaml_packages"] = len(yaml_packages)
report["groups"] = 0
for number, (package, links) in enumerate(groups.items(), 1):
print(f"[{number}/{len(groups)}] processing {len(links)} files for {package}")
version_groups = defaultdict(dict)
with tempfile.TemporaryDirectory(
dir=".git/pw-scratch/release-migration"
) as tmp:
tmp = Path(tmp)
for download_number, (url, filename, requires_python) in enumerate(
links, 1
):
print(f"downloading {filename}")
target = tmp / package / "downloads" / str(download_number) / filename
target.parent.mkdir(parents=True, exist_ok=True)
with session.get(url, stream=True, timeout=(30, 600)) as response:
response.raise_for_status()
with target.open("wb") as stream:
for chunk in response.iter_content(1024 * 1024):
stream.write(chunk)
with zipfile.ZipFile(target) as archive:
metadata_path = next(
name
for name in archive.namelist()
if name.endswith(".dist-info/METADATA")
)
message = Parser().parsestr(archive.read(metadata_path).decode())
project, version = message.get("Name"), message.get("Version")
if normalize(project) != package:
raise RuntimeError(f"package mismatch: {package}: {filename}")
digest = hashlib.sha256(target.read_bytes()).hexdigest()
fragment = urlparse(url).fragment
if fragment.startswith("sha256=") and digest != fragment.removeprefix(
"sha256="
):
raise RuntimeError(f"digest mismatch: {filename}")
file_data = {"filename": filename, "sha256": digest}
requires = message.get("Requires-Python") or requires_python
if requires:
file_data["requires-python"] = requires
previous = version_groups[version].get(filename)
if previous and previous["metadata"]["sha256"] != digest:
raise RuntimeError(f"conflicting duplicate filename: {filename}")
if previous:
previous["path"].unlink()
version_groups[version][filename] = {
"metadata": file_data,
"path": target,
}
report["groups"] += len(version_groups)
for version, file_map in sorted(version_groups.items()):
key = f"{package}=={version}"
paths = [entry["path"] for entry in file_map.values()]
try:
if key in state["completed"]:
print(f"already completed {key}")
continue
tag = f"{package}-v{version}-{state['timestamp']}"
title = f"{package}-v{version}"
files = [entry["metadata"] for entry in file_map.values()]
if not args.execute:
print(
f"validated {key} ({len(files)} files); would create {tag}"
)
continue
yaml_data = yaml.safe_load((DOCS / f"{package}.yaml").read_text())
version_item = next(
(
item
for item in yaml_data.get("versions", [])
if str(item.get("version")) == version
),
{},
)
gpl_sources = existing_gpl_sources(version_item)
if gpl_sources:
source = tmp / package / version / gpl_sources["filename"]
source.parent.mkdir(parents=True, exist_ok=True)
with session.get(
gpl_sources["url"], stream=True, timeout=(30, 600)
) as response:
response.raise_for_status()
with source.open("wb") as stream:
for chunk in response.iter_content(1024 * 1024):
stream.write(chunk)
paths.append(source)
gitlab_package_id = package_ids.get((package, version))
if gitlab_package_id is None:
raise RuntimeError(
f"missing GitLab package ID for {package}=={version}"
)
gitlab_package_url = f"{GITLAB_PACKAGE_UI}/{gitlab_package_id}"
migrated_at = datetime.strptime(
state["timestamp"], "%Y%m%d%H%M%S"
).replace(tzinfo=timezone.utc)
notes = (
f"Migrated from the [GitLab package]"
f"({gitlab_package_url}) on "
f"{migrated_at.strftime('%Y-%m-%d %H:%M:%S UTC')}."
)
existing = (
subprocess.run(
["gh", "release", "view", tag, "--repo", REPO],
capture_output=True,
text=True,
).returncode
== 0
)
if not existing:
run(
"gh",
"release",
"create",
tag,
"--repo",
REPO,
"--target",
"79acd1ea9648523a330db1767e9d645f14f51d5d",
"--title",
title,
"--notes",
notes,
"--draft",
"--latest=false",
)
run(
"gh",
"release",
"upload",
tag,
*map(str, paths),
"--repo",
REPO,
)
run(
"gh",
"release",
"edit",
tag,
"--repo",
REPO,
"--draft=false",
"--latest=false",
)
release = {
"tag": tag,
"files": sorted(files, key=lambda item: item["filename"]),
}
if gpl_sources:
release["gpl-sources"] = {
key: value
for key, value in gpl_sources.items()
if key != "url"
}
add_yaml_release(package, version, release)
state["completed"][key] = {"tag": tag, "files": len(files)}
save_state(state)
report["completed"].append(key)
print(f"published {tag} ({len(files)} files)")
finally:
for path in paths:
path.unlink(missing_ok=True)
REPORT.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n")
if not args.execute:
print("Dry run complete; pass --execute to publish releases and update YAML.")
if __name__ == "__main__":
try:
main()
except Exception as error:
print(f"error: {error}", file=sys.stderr)
raise
```
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
1 participant
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
Replace GitLab publishing with a reusable release workflow and generate the
/simpleAPI from package metadata.