From ea4050c2499142f8e630861a6f3cda0bd104eb68 Mon Sep 17 00:00:00 2001 From: Samk Date: Thu, 8 Oct 2026 17:41:51 +0530 Subject: [PATCH 1/3] Add mirror pipeline to collect Photon advisories Signed-off-by: Samk --- .github/workflows/sync.yml | 37 +++++++++ requirements.txt | 2 + sync.py | 150 +++++++++++++++++++++++++++++++++++++ 3 files changed, 189 insertions(+) create mode 100644 .github/workflows/sync.yml create mode 100644 requirements.txt create mode 100644 sync.py diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml new file mode 100644 index 0000000000..d1028f4dc5 --- /dev/null +++ b/.github/workflows/sync.yml @@ -0,0 +1,37 @@ +name: Daily sync of Photon Advisories + +on: + workflow_dispatch: + schedule: + - cron: '0 0 * * *' + +permissions: + contents: write + +jobs: + scheduled: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.10' + + - name: Install required packages + run: pip install -r requirements.txt + + - name: Run sync + run: python sync.py + + - name: Commit and push if it changed + run: |- + git config user.name "AboutCode Automation" + git config user.email "automation@aboutcode.org" + git add -A + timestamp=$(date -u) + git commit -m "$(echo -e "Sync Photon Advisories: $timestamp\n\nSigned-off-by: AboutCode Automation ")" || exit 0 + git push \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..d39c8799f0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +requests==2.32.5 +aboutcode.pipeline==0.2.1 \ No newline at end of file diff --git a/sync.py b/sync.py new file mode 100644 index 0000000000..3a65e7f40f --- /dev/null +++ b/sync.py @@ -0,0 +1,150 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import json +import re +import sys +from datetime import datetime, timezone +from pathlib import Path + +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry +from aboutcode.pipeline import BasePipeline, LoopProgress + +ROOT_PATH = Path(__file__).parent +ADVISORIES_PATH = ROOT_PATH / "advisories" +TIMESTAMPS_FILE = ROOT_PATH / "upstream_timestamps.json" + +YEAR_REGEX = re.compile(r'') +ENTRY_REGEX = re.compile(r'[^<]+\s+(\d{2}-[A-Za-z]{3}-\d{4} \d{2}:\d{2})') + +HEADERS = { + "Accept": "application/json, text/html", +} +REQUEST_TIMEOUT = 15 +BASE_URL = "https://packages.broadcom.com/photon/photon_cve_metadata/cve/" + +class PhotonAdvisoryMirror(BasePipeline): + @classmethod + def steps(cls): + return ( + cls.load_timestamps, + cls.create_session, + cls.collect_advisories, + cls.save_timestamps, + ) + + def load_timestamps(self): + self.timestamps = {} + if TIMESTAMPS_FILE.exists(): + with TIMESTAMPS_FILE.open("r") as f: + self.timestamps = json.load(f) + + def create_session(self): + retry = Retry( + total=3, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504] + ) + adapter = HTTPAdapter(max_retries=retry) + self.session = requests.Session() + self.session.headers.update(HEADERS) + self.session.mount("https://", adapter) + self.new_count = 0 + self.updated_count = 0 + + def collect_advisories(self): + """ + Discover missing or updated advisories across all year directories, queue them up and download them. + + Finds the filenames from the html response on the year pages and checks if they exist locally or are outdated. + + This method handles two cases: + 1. Missing advisories: If a file does not exist locally, it is added to ``updates_needed``. + 2. Updated advisories: If a file exists locally but its timestamp does not match the upstream timestamp, + it is added to ``updates_needed``. + + The ``updates_needed`` list stores tuples of the form (download_url, local_file_path, upstream_last_modified, relative_filepath). + """ + + self.log(f"Fetching root index from {BASE_URL}") + response = self.session.get(BASE_URL, timeout=REQUEST_TIMEOUT) + response.raise_for_status() + + years = YEAR_REGEX.findall(response.text) + updates_needed = [] + + for year in years: + year_url = f"{BASE_URL}{year}/" + year_path = ADVISORIES_PATH / year + year_path.mkdir(parents=True, exist_ok=True) + + response = self.session.get(year_url, timeout=REQUEST_TIMEOUT) + response.raise_for_status() + + for advisory_filename, upstream_last_modified in ENTRY_REGEX.findall(response.text): + local_file_path = year_path / advisory_filename + relative_filepath = f"{year}/{advisory_filename}" + + is_missing = not local_file_path.exists() + is_outdated = self.timestamps.get(relative_filepath) != upstream_last_modified + + if is_missing or is_outdated: + download_url = f"{year_url}{advisory_filename}" + updates_needed.append( + (download_url, local_file_path, upstream_last_modified, relative_filepath) + ) + + if not updates_needed: + self.log("No new or updated files found. Sync is already up to date.") + return + + self.log(f"Found {len(updates_needed)} files to download or update.") + + progress = LoopProgress(total_iterations=len(updates_needed), logger=self.log) + for download_url, local_file_path, upstream_last_modified, relative_filepath in progress.iter(updates_needed): + self.download_file(download_url, local_file_path, upstream_last_modified, relative_filepath) + + self.log(f"Sync complete. New: {self.new_count} | Updated: {self.updated_count}") + + def save_timestamps(self): + with TIMESTAMPS_FILE.open("w") as f: + json.dump(self.timestamps, f, indent=2) + + def download_file(self, download_url, local_file_path, upstream_last_modified, relative_filepath): + """ + Fetch a single advisory file and update its timestamp in the local state. + """ + try: + response = self.session.get(download_url, timeout=REQUEST_TIMEOUT) + response.raise_for_status() + except requests.exceptions.RequestException as error: + self.log(f"Failed to download {download_url}: {error}") + return + + is_new = not local_file_path.exists() + + with local_file_path.open("wb") as f: + f.write(response.content) + + if is_new: + self.new_count += 1 + else: + self.updated_count += 1 + + self.timestamps[relative_filepath] = upstream_last_modified + + def log(self, message): + now_local = datetime.now(timezone.utc).astimezone() + timestamp = now_local.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] + print(f"{timestamp} {message}") + +if __name__ == "__main__": + mirror = PhotonAdvisoryMirror() + status_code, error_message = mirror.execute() + if error_message: + print(error_message) + sys.exit(status_code) From 135e38514b3416ee28f0def0ce68fad574591a9e Mon Sep 17 00:00:00 2001 From: Samk Date: Thu, 15 Oct 2026 18:31:57 +0530 Subject: [PATCH 2/3] Use wget --mirror in sync workflow and remove sync.py Signed-off-by: Samk --- .github/workflows/sync.yml | 12 +-- requirements.txt | 2 - sync.py | 150 ------------------------------------- 3 files changed, 2 insertions(+), 162 deletions(-) delete mode 100644 requirements.txt delete mode 100644 sync.py diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index d1028f4dc5..cbf073e789 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -16,16 +16,8 @@ jobs: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: '3.10' - - - name: Install required packages - run: pip install -r requirements.txt - - name: Run sync - run: python sync.py + run: wget --mirror --no-parent --reject "index.html*" --no-host-directories --cut-dirs=3 -P advisories https://packages.broadcom.com/photon/photon_cve_metadata/cve/ - name: Commit and push if it changed run: |- @@ -34,4 +26,4 @@ jobs: git add -A timestamp=$(date -u) git commit -m "$(echo -e "Sync Photon Advisories: $timestamp\n\nSigned-off-by: AboutCode Automation ")" || exit 0 - git push \ No newline at end of file + git push diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index d39c8799f0..0000000000 --- a/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -requests==2.32.5 -aboutcode.pipeline==0.2.1 \ No newline at end of file diff --git a/sync.py b/sync.py deleted file mode 100644 index 3a65e7f40f..0000000000 --- a/sync.py +++ /dev/null @@ -1,150 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import json -import re -import sys -from datetime import datetime, timezone -from pathlib import Path - -import requests -from requests.adapters import HTTPAdapter -from urllib3.util.retry import Retry -from aboutcode.pipeline import BasePipeline, LoopProgress - -ROOT_PATH = Path(__file__).parent -ADVISORIES_PATH = ROOT_PATH / "advisories" -TIMESTAMPS_FILE = ROOT_PATH / "upstream_timestamps.json" - -YEAR_REGEX = re.compile(r'') -ENTRY_REGEX = re.compile(r'[^<]+\s+(\d{2}-[A-Za-z]{3}-\d{4} \d{2}:\d{2})') - -HEADERS = { - "Accept": "application/json, text/html", -} -REQUEST_TIMEOUT = 15 -BASE_URL = "https://packages.broadcom.com/photon/photon_cve_metadata/cve/" - -class PhotonAdvisoryMirror(BasePipeline): - @classmethod - def steps(cls): - return ( - cls.load_timestamps, - cls.create_session, - cls.collect_advisories, - cls.save_timestamps, - ) - - def load_timestamps(self): - self.timestamps = {} - if TIMESTAMPS_FILE.exists(): - with TIMESTAMPS_FILE.open("r") as f: - self.timestamps = json.load(f) - - def create_session(self): - retry = Retry( - total=3, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504] - ) - adapter = HTTPAdapter(max_retries=retry) - self.session = requests.Session() - self.session.headers.update(HEADERS) - self.session.mount("https://", adapter) - self.new_count = 0 - self.updated_count = 0 - - def collect_advisories(self): - """ - Discover missing or updated advisories across all year directories, queue them up and download them. - - Finds the filenames from the html response on the year pages and checks if they exist locally or are outdated. - - This method handles two cases: - 1. Missing advisories: If a file does not exist locally, it is added to ``updates_needed``. - 2. Updated advisories: If a file exists locally but its timestamp does not match the upstream timestamp, - it is added to ``updates_needed``. - - The ``updates_needed`` list stores tuples of the form (download_url, local_file_path, upstream_last_modified, relative_filepath). - """ - - self.log(f"Fetching root index from {BASE_URL}") - response = self.session.get(BASE_URL, timeout=REQUEST_TIMEOUT) - response.raise_for_status() - - years = YEAR_REGEX.findall(response.text) - updates_needed = [] - - for year in years: - year_url = f"{BASE_URL}{year}/" - year_path = ADVISORIES_PATH / year - year_path.mkdir(parents=True, exist_ok=True) - - response = self.session.get(year_url, timeout=REQUEST_TIMEOUT) - response.raise_for_status() - - for advisory_filename, upstream_last_modified in ENTRY_REGEX.findall(response.text): - local_file_path = year_path / advisory_filename - relative_filepath = f"{year}/{advisory_filename}" - - is_missing = not local_file_path.exists() - is_outdated = self.timestamps.get(relative_filepath) != upstream_last_modified - - if is_missing or is_outdated: - download_url = f"{year_url}{advisory_filename}" - updates_needed.append( - (download_url, local_file_path, upstream_last_modified, relative_filepath) - ) - - if not updates_needed: - self.log("No new or updated files found. Sync is already up to date.") - return - - self.log(f"Found {len(updates_needed)} files to download or update.") - - progress = LoopProgress(total_iterations=len(updates_needed), logger=self.log) - for download_url, local_file_path, upstream_last_modified, relative_filepath in progress.iter(updates_needed): - self.download_file(download_url, local_file_path, upstream_last_modified, relative_filepath) - - self.log(f"Sync complete. New: {self.new_count} | Updated: {self.updated_count}") - - def save_timestamps(self): - with TIMESTAMPS_FILE.open("w") as f: - json.dump(self.timestamps, f, indent=2) - - def download_file(self, download_url, local_file_path, upstream_last_modified, relative_filepath): - """ - Fetch a single advisory file and update its timestamp in the local state. - """ - try: - response = self.session.get(download_url, timeout=REQUEST_TIMEOUT) - response.raise_for_status() - except requests.exceptions.RequestException as error: - self.log(f"Failed to download {download_url}: {error}") - return - - is_new = not local_file_path.exists() - - with local_file_path.open("wb") as f: - f.write(response.content) - - if is_new: - self.new_count += 1 - else: - self.updated_count += 1 - - self.timestamps[relative_filepath] = upstream_last_modified - - def log(self, message): - now_local = datetime.now(timezone.utc).astimezone() - timestamp = now_local.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] - print(f"{timestamp} {message}") - -if __name__ == "__main__": - mirror = PhotonAdvisoryMirror() - status_code, error_message = mirror.execute() - if error_message: - print(error_message) - sys.exit(status_code) From 169d2ffa91f2402882c4d901f9cdda4a54abe7d0 Mon Sep 17 00:00:00 2001 From: Samk Date: Mon, 19 Oct 2026 14:57:11 +0530 Subject: [PATCH 3/3] Improve sync.yml Signed-off-by: Samk --- .github/workflows/sync.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index cbf073e789..d1c3af6382 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -17,13 +17,12 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Run sync - run: wget --mirror --no-parent --reject "index.html*" --no-host-directories --cut-dirs=3 -P advisories https://packages.broadcom.com/photon/photon_cve_metadata/cve/ + run: wget -r -l inf --no-parent --reject "index.html*" --no-host-directories --cut-dirs=3 -P advisories https://packages.broadcom.com/photon/photon_cve_metadata/cve/ - name: Commit and push if it changed run: |- git config user.name "AboutCode Automation" git config user.email "automation@aboutcode.org" git add -A - timestamp=$(date -u) - git commit -m "$(echo -e "Sync Photon Advisories: $timestamp\n\nSigned-off-by: AboutCode Automation ")" || exit 0 + git commit -m "$(echo -e "Sync Photon Advisories\n\nSigned-off-by: AboutCode Automation ")" || exit 0 git push