From a5bac68ede3282b0662a29d3f9328e5b2dc2593e Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Mon, 26 Oct 2026 12:15:45 -0700 Subject: [PATCH] Pin OpenVINO CI to the 2026.4.1 release instead of a deleted nightly (#23502) ## What is wrong today The OpenVINO test-harness jobs have failed at setup on every run since September 9, on main and `release/1.5`. The setup script downloads a pinned nightly that OpenVINO has since deleted. The server answers the missing file with an HTML page and HTTP 200, so `curl --fail` does not catch it, and the job dies in `tar`: ``` gzip: stdin: not in gzip format ``` The YOLO26 test script and its README pin a deleted nightly wheel too. The `openvino` pip extra, the docs and the runtime error message still cap OpenVINO below 2026.0.0, although CI runs the backend on 2026 releases. ## What this change does - Both Linux OpenVINO jobs install one pinned release, 2026.4.1, the latest. Before, the harness used that nightly and `test-openvino-linux` used 2026.0. - The download is checked against a pinned SHA-256, so a missing or replaced archive stops before `tar`. - `--nightly` stays for quick tests only. It installs the newest nightly from OpenVINO's `latest.txt`, checked against its published `.sha256`. No CI job passes it. - Pull requests that change `setup-openvino.sh` or `test_backend.sh` also run the harness. - The YOLO26 script and README install 2026.4.1 from PyPI. - The extra, docs and error message allow `openvino>=2025.1.0,<2027.0.0`. The extra still skips Python 3.15 (no build) and Linux aarch64 with Python 3.14 (builds need glibc 2.35, the ExecuTorch wheel 2.28). Not in this change: `nncf` stays at 3.1.0, and `release/1.5` still has the deleted pin. `resnet50`, `inception_v3` and `wide_resnet50_2` fail the harness tolerance, not from this change: on bfloat16 CPUs OpenVINO runs float32 models in bfloat16 unless given the f32 hint. On the same inputs, 2026.0.0, 2026.1.0 and 2026.4.1 give identical results. ## What was tested On Linux x86_64 (AMD EPYC, bfloat16): - The release and `--nightly` paths download, pass the check and extract; the deleted nightly URL and a swapped archive stop at the checksum. - Harness models: 29 of 35 pass; the 6 failures are those three models, static and dynamic. Operator sub-tests: 778 of 798 pass, same failures as 2026.1.0. - `test-openvino-linux` gives the same results with 2026.0.0 and 2026.4.1. Also: - A dry pip resolve of the extra picks 2026.4.1 on Linux x86_64, on Windows, and on Linux aarch64 with glibc 2.35 (Python 3.11 and 3.13). - With 2026.4.1, YOLO26 exports to the OpenVINO delegate (fp32, on macOS arm64). CI does not run the YOLO26 script. (cherry picked from commit 209be4bcd4e5cd11c0c10641af4ee2d24e6064af) --- .ci/scripts/setup-openvino.sh | 33 ++++++++++--------- .ci/scripts/test_backend.sh | 2 +- .ci/scripts/test_yolo26.sh | 3 +- .github/workflows/test-backend-openvino.yml | 2 ++ README-wheel.md | 4 +-- backends/openvino/README.md | 2 +- backends/openvino/runtime/OpenvinoBackend.cpp | 2 +- docs/source/build-run-openvino.md | 5 +-- docs/source/using-executorch-cpp.md | 3 +- examples/models/yolo26/README.md | 4 +-- pyproject.toml | 7 ++-- 11 files changed, 36 insertions(+), 31 deletions(-) diff --git a/.ci/scripts/setup-openvino.sh b/.ci/scripts/setup-openvino.sh index 6acc691649b..d2ade9954d5 100755 --- a/.ci/scripts/setup-openvino.sh +++ b/.ci/scripts/setup-openvino.sh @@ -18,26 +18,27 @@ for arg in "$@"; do esac done -# Download and install OpenVINO from release packages -OPENVINO_VERSION="2026.0" -OPENVINO_BUILD="2026.0.0.20965.c6d6a13a886" -OPENVINO_STABLE_URL="https://storage.openvinotoolkit.org/repositories/openvino/packages/${OPENVINO_VERSION}/linux/openvino_toolkit_ubuntu22_${OPENVINO_BUILD}_x86_64.tgz" - -OPENVINO_NIGHTLY_BUILD_ID="2026.1.0-21310-c694fbc2b6d" -OPENVINO_NIGHTLY_BUILD="2026.1.0.dev20260312" -OPENVINO_NIGHTLY_URL="https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/${OPENVINO_NIGHTLY_BUILD_ID}/openvino_toolkit_ubuntu22_${OPENVINO_NIGHTLY_BUILD}_x86_64.tgz" - +OPENVINO_PACKAGES_URL="https://storage.openvinotoolkit.org/repositories/openvino/packages" if [ "${USE_NIGHTLY}" = true ]; then - OPENVINO_URL="${OPENVINO_NIGHTLY_URL}" - OPENVINO_EXTRACTED_DIR="openvino_toolkit_ubuntu22_${OPENVINO_NIGHTLY_BUILD}_x86_64" - echo "Using OpenVINO nightly build: ${OPENVINO_NIGHTLY_BUILD_ID}" + # Only for quick testing. OpenVINO deletes nightlies after a few months, so + # take the newest one instead of pinning a build that will disappear. + OPENVINO_URL=$(curl -fsSL --retry 3 --retry-all-errors "${OPENVINO_PACKAGES_URL}/nightly/latest.txt" | sed -n 's/^ubuntu22_x86_64://p') + : "${OPENVINO_URL:?could not get the ubuntu22_x86_64 nightly from latest.txt}" + OPENVINO_SHA256=$(curl -fsSL --retry 3 --retry-all-errors "${OPENVINO_URL}.sha256" | sed -nE 's/^([0-9a-f]{64}).*/\1/p') + : "${OPENVINO_SHA256:?could not get the SHA-256 of the nightly from its .sha256 file}" else - OPENVINO_URL="${OPENVINO_STABLE_URL}" - OPENVINO_EXTRACTED_DIR="openvino_toolkit_ubuntu22_${OPENVINO_BUILD}_x86_64" - echo "Using OpenVINO stable release: ${OPENVINO_BUILD}" + OPENVINO_VERSION="2026.4.1" + OPENVINO_BUILD="2026.4.1.22982.07f9c262b05" + OPENVINO_SHA256="e2541afe65a81d5aabe981f9a86ad2c7efe557922b71e5649b6aafffec7c982b" + OPENVINO_URL="${OPENVINO_PACKAGES_URL}/${OPENVINO_VERSION}/linux/openvino_toolkit_ubuntu22_${OPENVINO_BUILD}_x86_64.tgz" fi +OPENVINO_EXTRACTED_DIR=$(basename "${OPENVINO_URL}" .tgz) +echo "Using OpenVINO: ${OPENVINO_EXTRACTED_DIR}" -curl -Lo /tmp/openvino_toolkit.tgz --retry 3 --retry-all-errors --fail ${OPENVINO_URL} +curl -Lo /tmp/openvino_toolkit.tgz --retry 3 --retry-all-errors --fail "${OPENVINO_URL}" +# The storage server answers a missing file with an HTML page and HTTP 200, so +# compare the download with the expected hash before extracting it. +echo "${OPENVINO_SHA256} /tmp/openvino_toolkit.tgz" | sha256sum -c - tar -xzf /tmp/openvino_toolkit.tgz mv "${OPENVINO_EXTRACTED_DIR}" openvino diff --git a/.ci/scripts/test_backend.sh b/.ci/scripts/test_backend.sh index 8bfe33333a8..51a49c7f6d2 100755 --- a/.ci/scripts/test_backend.sh +++ b/.ci/scripts/test_backend.sh @@ -119,7 +119,7 @@ fi if [[ "$FLOW" == *openvino* ]]; then # Setup OpenVINO environment - source .ci/scripts/setup-openvino.sh --nightly + source .ci/scripts/setup-openvino.sh EXTRA_BUILD_ARGS+=" -DEXECUTORCH_BUILD_OPENVINO=ON" fi diff --git a/.ci/scripts/test_yolo26.sh b/.ci/scripts/test_yolo26.sh index 08925dc3dbb..38ac2076560 100755 --- a/.ci/scripts/test_yolo26.sh +++ b/.ci/scripts/test_yolo26.sh @@ -76,8 +76,7 @@ if [[ "${MODE}" =~ .*openvino.* ]]; then OPENVINO=ON TARGET_LIBS="$TARGET_LIBS openvino_backend " - # Install specific OpenVINO runtime from pip. - $PYTHON_EXECUTABLE -m pip install --pre openvino==2026.1.0.dev20260131 --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + $PYTHON_EXECUTABLE -m pip install openvino==2026.4.1 $PYTHON_EXECUTABLE -m pip install -r backends/openvino/requirements.txt # Set OPENVINO_LIB_PATH so the C++ demo runner can also find libopenvino_c.so. diff --git a/.github/workflows/test-backend-openvino.yml b/.github/workflows/test-backend-openvino.yml index cff7f8ba6e9..44a9b358f1c 100644 --- a/.github/workflows/test-backend-openvino.yml +++ b/.github/workflows/test-backend-openvino.yml @@ -10,6 +10,8 @@ on: - ciflow/nightly/* pull_request: paths: + - .ci/scripts/setup-openvino.sh + - .ci/scripts/test_backend.sh - .github/workflows/test-backend-openvino.yml - .github/workflows/_test_backend.yml workflow_dispatch: diff --git a/README-wheel.md b/README-wheel.md index c76f6819010..5245e312bcd 100644 --- a/README-wheel.md +++ b/README-wheel.md @@ -37,8 +37,8 @@ to run ExecuTorch `.pte` files, with some restrictions: * \[Linux and Windows] [OpenVINO](docs/source/build-run-openvino.md) backend is also linked into the prebuilt module. OpenVINO requires the runtime to be installed separately: `pip install executorch[openvino]`. For now the extra installs no runtime on Python 3.15, - or on Linux aarch64 with Python 3.14, because the OpenVINO versions it supports publish no - build there. + or on Linux aarch64 with Python 3.14. OpenVINO publishes no build for Python 3.15, and its + aarch64 builds for Python 3.14 need glibc 2.35 or newer. On Linux, macOS and Windows the runtime, kernels and backends also ship as shared libraries with headers and a CMake package, so a C++ application can link them without building diff --git a/backends/openvino/README.md b/backends/openvino/README.md index 56d2f13383f..331aebf87ff 100644 --- a/backends/openvino/README.md +++ b/backends/openvino/README.md @@ -20,7 +20,7 @@ On Linux and Windows, the OpenVINO backend is included in the ExecuTorch pip whe pip install executorch[openvino] ``` -For now the extra installs no runtime on Python 3.15, or on Linux aarch64 with Python 3.14, because the OpenVINO versions it supports publish no build there. +For now the extra installs no runtime on Python 3.15, or on Linux aarch64 with Python 3.14. OpenVINO publishes no build for Python 3.15, and its aarch64 builds for Python 3.14 need glibc 2.35 or newer. The backend automatically discovers the OpenVINO C library from the pip-installed package, so no library search path setup is needed. diff --git a/backends/openvino/runtime/OpenvinoBackend.cpp b/backends/openvino/runtime/OpenvinoBackend.cpp index 7fec8fd150f..0689e5d4680 100644 --- a/backends/openvino/runtime/OpenvinoBackend.cpp +++ b/backends/openvino/runtime/OpenvinoBackend.cpp @@ -212,7 +212,7 @@ bool OpenvinoBackend::ensure_loaded() const { Error, "OpenVINO runtime not found (loading '%s' failed: %s). " "Ensure '%s' can be found (%s), or install with: " - "pip install \"openvino>=2025.1.0,<2026.0.0\"", + "pip install \"openvino>=2025.1.0,<2027.0.0\"", to_utf8(effective_path).c_str(), error.c_str(), to_utf8(kDefaultLibName).c_str(), diff --git a/docs/source/build-run-openvino.md b/docs/source/build-run-openvino.md index d749e286c26..69a73672ac1 100644 --- a/docs/source/build-run-openvino.md +++ b/docs/source/build-run-openvino.md @@ -44,7 +44,8 @@ pip install executorch[openvino] ``` For now the extra installs no runtime on Python 3.15, or on Linux aarch64 with Python -3.14, because the OpenVINO versions it supports publish no build there. +3.14. OpenVINO publishes no build for Python 3.15, and its aarch64 builds for Python 3.14 +need glibc 2.35 or newer. The backend automatically discovers the OpenVINO C library from the pip-installed package — no environment variable setup is needed. @@ -60,7 +61,7 @@ If you need a custom build (different platform, custom configuration, etc.), fol Before you begin, ensure you have the OpenVINO runtime installed and configured on your system. For most users, we recommend installing it via pip: ```bash -pip install "openvino>=2025.1.0,<2026.0.0" +pip install "openvino>=2025.1.0,<2027.0.0" ``` Alternatively, you can build OpenVINO from source or follow the official [OpenVINO installation guide](https://docs.openvino.ai/2025/get-started/install-openvino.html) for platform-specific options: diff --git a/docs/source/using-executorch-cpp.md b/docs/source/using-executorch-cpp.md index 8b0c8df29b4..2391584196f 100644 --- a/docs/source/using-executorch-cpp.md +++ b/docs/source/using-executorch-cpp.md @@ -273,7 +273,8 @@ Without it the delegate still registers and the program still links, and the fai when the model is loaded. For now the openvino extra installs no runtime on Python 3.15, or on Linux aarch64 with -Python 3.14, because the OpenVINO versions it supports publish no build there. +Python 3.14. OpenVINO publishes no build for Python 3.15, and its aarch64 builds for Python +3.14 need glibc 2.35 or newer. The MLX delegate has a similar requirement. Its Metal kernels live in a separate `mlx.metallib` file, which MLX looks for next to whichever library holds MLX code. The wheel ships it beside the diff --git a/examples/models/yolo26/README.md b/examples/models/yolo26/README.md index f853718ad65..77cfbca8ebe 100644 --- a/examples/models/yolo26/README.md +++ b/examples/models/yolo26/README.md @@ -16,9 +16,9 @@ To install ExecuTorch, follow this [guide](https://pytorch.org/executorch/stable - [OpenVINO backend installation guide](../../../backends/openvino/README.md#quick-start-pip-wheel) -> **_NOTE:_** Please install the OpenVINO nightly version that supports the YOLO26 model. +> **_NOTE:_** Please install the OpenVINO version that supports the YOLO26 model. ```bash -python -m pip install --pre openvino==2026.1.0.dev20260131 --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly +python -m pip install openvino==2026.4.1 ``` - [XNNPACK backend installation guide](https://pytorch.org/executorch/stable/tutorial-xnnpack-delegate-lowering.html#running-the-xnnpack-model-with-cmake) diff --git a/pyproject.toml b/pyproject.toml index 7b865171146..116552df1ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,9 +86,10 @@ ethos_u = [ "tosa-tools==2026.5.0", ] openvino = [ - # The OpenVINO versions in this range publish no build for Python 3.15, or for Linux - # aarch64 on Python 3.14, so without these bounds pip fails to install the extra there. - "openvino>=2025.1.0,<2026.0.0; (platform_system == 'Linux' or (platform_system == 'Windows' and platform_machine == 'AMD64')) and python_version < '3.15' and (platform_machine != 'aarch64' or python_version < '3.14')", + # OpenVINO publishes no build for Python 3.15, and its Linux aarch64 builds for Python + # 3.14 need glibc 2.35 while the wheel needs only 2.28. Without these bounds pip fails to + # install the extra on Python 3.15 and on those older aarch64 systems. + "openvino>=2025.1.0,<2027.0.0; (platform_system == 'Linux' or (platform_system == 'Windows' and platform_machine == 'AMD64')) and python_version < '3.15' and (platform_machine != 'aarch64' or python_version < '3.14')", ] [project.urls]