executorch-test-linux.yml

name: ExecuTorch Delegate Linux Test

on:
  workflow_call:
    inputs:
      build-matrix:
        description: "Build matrix to utilize"
        default: ""
        type: string
      repository:
        description: 'Repository to checkout, defaults to ""'
        default: ""
        type: string
      ref:
        description: 'Reference to checkout, defaults to ""'
        default: ""
        type: string
      test-infra-repository:
        description: "Test infra repository to use"
        default: "pytorch/test-infra"
        type: string
      test-infra-ref:
        description: "Test infra reference to use"
        default: ""
        type: string

jobs:
  filter-matrix:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.generate.outputs.matrix }}
    steps:
      - uses: actions/setup-python@v6
        with:
          python-version: "3.11"
      - uses: actions/checkout@v6
      - name: Filter matrix
        id: generate
        run: |
          set -eou pipefail
          MATRIX_BLOB=${{ toJSON(inputs.build-matrix) }}
          LIMIT_PR=${{ (github.event_name == 'push' || github.event_name == 'schedule') && 'false' || 'true' }}
          MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --executorch-runtime --use-rtx false --limit-pr-builds "${LIMIT_PR}" --matrix "${MATRIX_BLOB}")"
          echo "${MATRIX_BLOB}"
          echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}"

  test:
    needs: filter-matrix
    uses: ./.github/workflows/linux-test.yml
    with:
      job-name: executorch-runtime-test
      repository: ${{ inputs.repository }}
      ref: ${{ inputs.ref }}
      test-infra-repository: ${{ inputs.test-infra-repository }}
      test-infra-ref: ${{ inputs.test-infra-ref }}
      build-matrix: ${{ needs.filter-matrix.outputs.matrix }}
      pre-script: packaging/pre_build_script.sh
      script: |
        set -euo pipefail
        mkdir -p "${RUNNER_TEMP}/bin"
        curl -L "https://github.com/bazelbuild/bazelisk/releases/download/v1.26.0/bazelisk-linux-amd64" \
          -o "${RUNNER_TEMP}/bin/bazel"
        chmod +x "${RUNNER_TEMP}/bin/bazel"
        export PATH="${RUNNER_TEMP}/bin:${PATH}"

        # ExecuTorch's CUDA wheels live only on the PyTorch indexes: a development build on the
        # nightly one, a release on the test one. No --pre: a specifier naming a prerelease admits
        # prereleases by itself, and --pre would apply to every other requirement in the same
        # command too. The version comes from the requirement file so a bump changes no file in
        # this directory.
        python -m pip install pyyaml "wheel>=0.40" \
          --extra-index-url "https://download.pytorch.org/whl/nightly/${CU_VERSION}" \
          --extra-index-url "https://download.pytorch.org/whl/test/${CU_VERSION}" \
          -r packaging/executorch_pin.txt

        # The shared artifact carries the matching core and companion wheels.
        shopt -s nullglob
        runtime_wheels=(/opt/torch-tensorrt-builds/torch_tensorrt_executorch_runtime*.whl)
        echo "Downloaded wheel artifact contents:"
        find /opt/torch-tensorrt-builds -maxdepth 1 -type f -name '*.whl' -printf '  %f (%s bytes)\n' | sort
        if [[ ${#runtime_wheels[@]} -ne 1 ]]; then
          echo "::error::Expected exactly one ExecuTorch runtime wheel in the standard artifact; found ${#runtime_wheels[@]}" >&2
          exit 1
        fi
        echo "Installing ExecuTorch runtime wheel: ${runtime_wheels[0]}"
        python -m pip install --no-deps "${runtime_wheels[0]}"
        # Check stock backends too, since delegated programs can fall back to them.
        # Run directly so a native crash fails the step; gdb is only for a backtrace.
        ulimit -c unlimited || true
        # sys.exit, not assert: under PYTHONOPTIMIZE or python -O every assert is compiled out and
        # this check would pass over a runtime with no backends registered at all.
        #
        # Importing the delegate package is the whole registration step; it exposes no runtime API,
        # so the registry is queried through ExecuTorch's own Runtime. Importing it for its side
        # effect only is exactly what a user does, so this also proves the side effect works.
        runtime_check='import sys, torch; print(torch.__version__, torch.version.cuda); import torch_tensorrt_executorch_runtime; from torch_tensorrt_executorch_runtime import BACKEND_NAME; from executorch.runtime import Runtime; registry = Runtime.get().backend_registry; missing = [n for n in (BACKEND_NAME, "XnnpackBackend", "CudaBackend") if not registry.is_available(n)]; sys.exit("FATAL: backends not registered: " + ", ".join(missing)) if missing else None'
        check_status=0
        PYTHONFAULTHANDLER=1 python -u -X faulthandler -c "${runtime_check}" ||
          check_status=$?
        if [[ "${check_status}" -ne 0 ]]; then
          echo "::error::native runtime check failed with status ${check_status}"
          if command -v gdb >/dev/null 2>&1; then
            gdb --batch -ex "set pagination off" -ex run \
              -ex "thread apply all bt" \
              --args python -u -X faulthandler -c "${runtime_check}" || true
          fi
          exit "${check_status}"
        fi

        # Resolve the user-facing range without reusing the build's installed pin.
        python -m venv "${RUNNER_TEMP}/range-check-venv"
        # pin-check: range-ok
        "${RUNNER_TEMP}/range-check-venv/bin/python" -m pip install --no-deps --extra-index-url "https://download.pytorch.org/whl/nightly/${CU_VERSION}" --extra-index-url "https://download.pytorch.org/whl/test/${CU_VERSION}" -r packaging/executorch_range.txt

        # pip does not fail when an extra cannot be satisfied; it quietly picks an older
        # torch-tensorrt. So check here that the executorch extra requires exactly the companion
        # built beside it when this run's channel also carries the ExecuTorch build that companion
        # pins (nightly for a development build, test for a release), that it requires none
        # otherwise, and that pip resolves that extra for the version just built.
        python - <<'PY'
        import importlib.metadata as metadata
        import os
        import sys

        from packaging.requirements import Requirement

        companion = metadata.version("torch-tensorrt-executorch-runtime")
        (executorch,) = [
            r
            for r in map(Requirement, metadata.requires("torch-tensorrt-executorch-runtime") or [])
            if r.name == "executorch"
        ]
        executorch_channel = "nightly" if ".dev" in str(executorch.specifier) else "test"
        pins = [
            r
            for r in map(Requirement, metadata.requires("torch-tensorrt") or [])
            if r.name == "torch-tensorrt-executorch-runtime"
            and r.marker is not None
            and r.marker.evaluate({"extra": "executorch"})
        ]
        expected = [f"=={companion}"] if os.environ["CHANNEL"] == executorch_channel else []
        if [str(r.specifier) for r in pins] != expected:
            sys.exit(f"the executorch extra requires {pins}, not {expected} (companion {companion}, channel {os.environ['CHANNEL']}, {executorch})")
        PY
        built="$(python -c 'import importlib.metadata as m; print(m.version("torch-tensorrt"))')"
        python -m pip install --pre --dry-run --find-links /opt/torch-tensorrt-builds \
          --extra-index-url "https://download.pytorch.org/whl/nightly/${CU_VERSION}" \
          --extra-index-url "https://download.pytorch.org/whl/test/${CU_VERSION}" \
          --extra-index-url https://pypi.nvidia.com \
          "torch-tensorrt[executorch]==${built}"

        source tests/py/utils/ci_helpers.sh
        trt_tier_executorch

        bazel build //:libtorchtrt --compilation_mode opt --config=linux
        # Execute the delegate C++ unit tests as well as building the Python
        # examples below. Bazel's cc_import provides unversioned CUDA/TensorRT
        # libraries, while the loader resolves versioned sonames, so expose the
        # directories containing those sonames to the test process.
        bazel_external="$(bazel info output_base)/external"
        for _soname in libnvinfer.so libcudart.so; do
          _dir="$(
            find -L "${bazel_external}" -path "*/lib*/${_soname}.*" -printf '%h\n' 2>/dev/null |
              sort -u | head -n1
          )"
          if [[ -z "${_dir}" ]]; then
            echo "Could not locate a versioned ${_soname} under ${bazel_external}" >&2
            exit 1
          fi
          export LD_LIBRARY_PATH="${_dir}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
        done
        executorch_cmake_location="$(bazel query @executorch//:executorch/CMakeLists.txt --output=location)"
        export EXECUTORCH_SOURCE_DIR="$(dirname "${executorch_cmake_location%%:*}")"
        export EXECUTORCH_ROOT="${EXECUTORCH_SOURCE_DIR}"
        python examples/torchtrt_executorch_example/export_static_shape.py \
          --model_path="${RUNNER_TEMP}/torchtrt-python.pte"
        python examples/torchtrt_executorch_example/export_kv_cache_decode.py \
          --model_path="${RUNNER_TEMP}/torchtrt-kv-cache-decode.pte"
        python examples/torchtrt_executorch_example/export_kv_cache_decode.py \
          --model_path="${RUNNER_TEMP}/torchtrt-kv-cache-decode-zero-copy.pte" \
          --zero_copy
        python examples/torchtrt_executorch_example/export_coalesced.py \
          --model_path="${RUNNER_TEMP}/torchtrt-coalesced.pte"
        python examples/torchtrt_executorch_example/export_device_resident.py \
          --model_path="${RUNNER_TEMP}/torchtrt-device-resident.pte"
        .github/scripts/verify-executorch-reference-runner.sh \
          "${RUNNER_TEMP}/torchtrt-python.pte" \
          --coalesced="${RUNNER_TEMP}/torchtrt-coalesced.pte" \
          "${RUNNER_TEMP}/torchtrt-kv-cache-decode.pte" \
          "${RUNNER_TEMP}/torchtrt-kv-cache-decode-zero-copy.pte"
        python examples/executorch_reference_runner/load_model.py \
          --model_path="${RUNNER_TEMP}/torchtrt-python.pte" --num_runs=1
        # The device-resident program is deliberately NOT handed to the C++
        # reference runner above: that runner feeds host tensors, which this
        # program's boundary contract forbids. Its Python runner passes CUDA
        # tensors and asserts the export flags took and the values match.
        python examples/executorch_reference_runner/load_model_device_resident.py \
          --model_path="${RUNNER_TEMP}/torchtrt-device-resident.pte" --num_runs=2
        # The only consumer that links the delegate the way an application does: out of the
        # installed wheel, found by name, through ExecuTorch's module extension. The reference
        # runner above drives the low level Method API and builds the delegate from source, so
        # nothing else here exercises the CMake package the wheel ships, or that C++ API.
        executorch_share="$(python -c 'import importlib.metadata as m, pathlib; print(pathlib.Path(str(m.distribution("executorch").locate_file("executorch"))) / "share" / "cmake")')"
        delegate_cmake="$(TORCH_TENSORRT_SKIP_DELEGATE_REGISTRATION=1 python -c 'import pathlib, torch_tensorrt_executorch_runtime as r; print(pathlib.Path(r.__file__).parent / "lib" / "cmake" / "executorch_backend_tensorrt")')"
        cmake -S examples/executorch_module_consumer -B "${RUNNER_TEMP}/module-consumer" \
          -DCMAKE_PREFIX_PATH="${executorch_share};${delegate_cmake}"
        cmake --build "${RUNNER_TEMP}/module-consumer" -j"$(nproc)"
        # Host outputs: the program plans them, so forward() alone is enough.
        "${RUNNER_TEMP}/module-consumer/executorch_module_consumer" \
          --model_path="${RUNNER_TEMP}/torchtrt-coalesced.pte" --num_runs=2
        # Device outputs: the program leaves them to the caller, so one is supplied first.
        "${RUNNER_TEMP}/module-consumer/executorch_module_consumer" \
          --model_path="${RUNNER_TEMP}/torchtrt-device-resident.pte" --device_io --num_runs=2
        # The backend C++ suite runs here, at the end, and not next to the Bazel
        # build above, because `set -e` ends this script at the first failure and
        # everything between the two would be traded away for it: the three
        # exports, verify-executorch-reference-runner.sh -- which unpacks the
        # release archive and builds the packaged CMake backend out of it, and so
        # is what says the archive still carries every header that backend needs
        # -- and load_model.py. Run last, a suite that fails because the device
        # went missing or flaky costs none of them.
        #
        # --test_summary=detailed names every case that skipped, in Bazel's own
        # summary rather than in the test log, which --test_output=errors prints
        # nothing of for a target that passes. Cases in that suite skip for reasons
        # the variable below does not cover -- its own coverage note lists them --
        # and without those names a run in which the pool's growth and
        # allocation-failure paths never executed looks like one in which they did.
        #
        # TORCHTRT_EXECUTORCH_REQUIRE_CUDA turns a case that skips for want of a
        # CUDA device into a failure. Every row of this job's matrix is a CUDA row
        # -- filter-matrix.py drops any whose desired_cuda is not a TensorRT CUDA
        # version -- and linux-test.yml starts the container with `--gpus all` for
        # those, so a skip here means the device went missing rather than that the
        # runner never had one. Without the variable every case that needs a
        # device skips, its target exits zero and is reported as a passing target
        # that covered nothing -- 27 of the suite's 96 cases, the whole of
        # test_shared_scratch_backend.
        bazel test //tests/cpp/executorch:executorch_backend_tests \
          --compilation_mode opt --config=linux --test_output=errors \
          --test_summary=detailed \
          --test_env=LD_LIBRARY_PATH \
          --test_env=TORCHTRT_EXECUTORCH_REQUIRE_CUDA=1