Tmp

.ci/scripts/test-native.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -euxo pipefail
+
+component="${1:?Usage: test-native.sh <runtime|ggml|vulkan> [cmake options]}"
+shift
+build_dir="cmake-out/native-${component}"
+build_jobs="${CMAKE_BUILD_PARALLEL_LEVEL:-$(nproc)}"
+cmake_args=()
+targets=()
+
+case "${component}" in
+  runtime)
+    targets+=(native_runtime_test)
+    test_regex='^native_runtime_test$'
+    ;;
+  ggml)
+    cmake_args+=(
+      -DEXECUTORCH_BUILD_NATIVE_GGML=ON
+      -DEXECUTORCH_BUILD_NATIVE_LLM_RUNNER=ON
+      -DGGML_NATIVE=OFF
+    )
+    targets+=(run_hf_llm_ggml native_ggml_engine_test native_llm_runner_utils_test)
+    test_regex='^native_(ggml_engine|llm_runner_utils)_test$'
+    ;;
+  vulkan)
+    cmake_args+=(
+      -DEXECUTORCH_BUILD_NATIVE_VULKAN=ON
+      -DEXECUTORCH_BUILD_NATIVE_VISION_RUNNER=ON
+      -DEXECUTORCH_BUILD_NATIVE_LLM_RUNNER=ON
+      -DEXECUTORCH_VULKAN_SHADER_COMPILE_NTHREADS="${build_jobs}"
+    )
+    targets+=(native_vulkan_engine run_classification run_hf_llm_vulkan native_vulkan_passes_test native_runner_utils_test)
+    test_regex='^native_(vulkan_passes|runner_utils)_test$'
+    ;;
+  *)
+    echo "Unknown native component: ${component}" >&2
+    exit 1
+    ;;
+esac
+
+cmake -S . -B "${build_dir}" -G Ninja \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DPYTHON_EXECUTABLE="$(command -v python)" \
+  -DEXECUTORCH_BUILD_TESTS=ON \
+  -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
+  -DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
+  -DEXECUTORCH_BUILD_EXECUTOR_RUNNER=OFF \
+  -DEXECUTORCH_BUILD_NATIVE=ON \
+  "${cmake_args[@]}" "$@"
+cmake --build "${build_dir}" \
+  --parallel "${build_jobs}" \
+  --target "${targets[@]}"
+ctest --test-dir "${build_dir}/backends/native/test" \
+  -R "${test_regex}" --output-on-failure --no-tests=error --timeout 120
+
+if [[ "${component}" == "ggml" ]]; then
+  PYTHONPATH="${PWD}/src:${PYTHONPATH:-}" \
+    FLATC_EXECUTABLE="${FLATC_EXECUTABLE:-${PWD}/${build_dir}/third-party/flatc_ep/bin/flatc}" \
+    python .ci/scripts/test_native_llm_runner.py \
+      --runner "${build_dir}/backends/native/examples/hf_llm/runtime/run_hf_llm_ggml"
+fi

.ci/scripts/test_native_llm_runner.py

@@ -0,0 +1,109 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Compare GGML LLM runner prefill and decoding with a tiny eager HF Llama."""
+
+import argparse
+import base64
+import re
+import subprocess
+import tempfile
+from pathlib import Path
+
+import torch
+from executorch.backends.native.examples.hf_llm.export_hf_llm import LastTokenLM
+from executorch.exir.native import to_native
+from transformers import LlamaConfig, LlamaForCausalLM
+
+
+@torch.no_grad()
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--runner", required=True, type=Path)
+    args = parser.parse_args()
+
+    torch.manual_seed(0)
+    torch.set_num_threads(4)
+    max_seq_length = 48
+    max_new_tokens = 8
+    config = LlamaConfig(
+        vocab_size=512,
+        hidden_size=32,
+        intermediate_size=64,
+        num_hidden_layers=2,
+        num_attention_heads=4,
+        num_key_value_heads=2,
+        max_position_embeddings=max_seq_length,
+        bos_token_id=256,
+        eos_token_id=257,
+        attn_implementation="sdpa",
+    )
+    model = LlamaForCausalLM(config).eval()
+    model.generation_config.use_cache = True
+    model.generation_config.cache_implementation = "static"
+    wrapper = LastTokenLM(model, max_seq_length).eval()
+    seq = torch.export.Dim("seq", min=1, max=max_seq_length - 1)
+    exported = torch.export.export(
+        wrapper,
+        (torch.zeros((1, 6), dtype=torch.long), torch.arange(6)),
+        dynamic_shapes={"input_ids": {1: seq}, "cache_position": {0: seq}},
+        strict=True,
+    )
+
+    with tempfile.TemporaryDirectory() as directory:
+        root = Path(directory)
+        package = root / "llama.ptn"
+        to_native(exported).save(str(package))
+        tokenizer = root / "tokenizer.model"
+        # The byte vocabulary puts Llama's BOS at 256, followed by 255 specials.
+        tokenizer.write_text(
+            "".join(
+                f"{base64.b64encode(bytes([i])).decode()} {i}\n" for i in range(256)
+            )
+        )
+        prompt = "abc"
+        unreachable_eos_id = config.vocab_size
+        tokens = [config.bos_token_id, *prompt.encode()]
+        expected = []
+        # Recompute the full prefix without a cache for an independent reference.
+        for _ in range(max_new_tokens):
+            logits = model(torch.tensor([tokens]), use_cache=False).logits
+            token = int(logits[0, -1].argmax())
+            expected.append(token)
+            tokens.append(token)
+
+        result = subprocess.run(
+            [
+                str(args.runner.resolve()),
+                "--device=cpu",
+                "--cpu_threads=4",
+                f"--package={package}",
+                f"--tokenizer={tokenizer}",
+                f"--prompt={prompt}",
+                f"--eos_ids={unreachable_eos_id}",
+                f"--max_new_tokens={max_new_tokens}",
+                f"--max_seq_length={max_seq_length}",
+                "--print_token_ids",
+            ],
+            capture_output=True,
+            text=True,
+            errors="replace",
+            timeout=120,
+        )
+        print(result.stdout, end="")
+        print(result.stderr, end="")
+        result.check_returncode()
+        match = re.search(r"^token_ids: ([0-9,]+)$", result.stdout, re.MULTILINE)
+        if match is None:
+            raise AssertionError("Runner did not report generated token IDs")
+        actual = [int(token) for token in match[1].split(",")]
+        if actual != expected:
+            raise AssertionError(f"Generated tokens {actual} != eager {expected}")
+        print(f"GGML HF LLM runner matched all {max_new_tokens} eager token IDs")
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/cuda.yml

@@ -43,6 +43,10 @@ on:
       - examples/models/muse-glimmer/**
       - extension/pybindings/**
       - runtime/__init__.py
+      - exir/passes/convert_constant_dim_order_pass.py
+      - exir/passes/insert_write_back_for_buffers_pass.py
+      - exir/passes/propagate_device_config.py
+      - exir/passes/propagate_device_pass.py
   workflow_dispatch:
     inputs:
       models:
@@ -469,8 +473,9 @@ jobs:
   unittest-cuda:
     name: unittest-cuda (${{ matrix.shard }})
     needs: [changed-files, run-decision]
-    # This job runs the pybindings device test, so it also has to fire on the
-    # two pybindings paths the workflow filter lists.
+    # This job runs the pybindings device test and the device placement test,
+    # so it also has to fire on the pybindings and device pass paths the
+    # workflow filter lists.
     if: |
       contains(needs.changed-files.outputs.changed-files, 'backends/cuda') ||
       contains(needs.changed-files.outputs.changed-files, 'backends/aoti') ||
@@ -481,6 +486,10 @@ jobs:
       contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_model_e2e.sh') ||
       contains(needs.changed-files.outputs.changed-files, 'extension/pybindings') ||
       contains(needs.changed-files.outputs.changed-files, 'runtime/__init__.py') ||
+      contains(needs.changed-files.outputs.changed-files, 'exir/passes/convert_constant_dim_order_pass.py') ||
+      contains(needs.changed-files.outputs.changed-files, 'exir/passes/insert_write_back_for_buffers_pass.py') ||
+      contains(needs.changed-files.outputs.changed-files, 'exir/passes/propagate_device_config.py') ||
+      contains(needs.changed-files.outputs.changed-files, 'exir/passes/propagate_device_pass.py') ||
       needs.run-decision.outputs.is-full-run == 'true'
     # kernel: Triton kernels and their autotuning. system: lowering passes, op
     # dispatch, export/runtime, and model tests. Together they do not fit the
@@ -567,6 +576,21 @@ jobs:
             echo "::error::a pybindings device test skipped in the CUDA job, which means the CUDA backend or a visible GPU is missing"
             exit 1
           fi
+
+          # Same for the device placement test, which skips without a GPU.
+          set +e
+          python -m pytest backends/cuda/tests/test_device_placement.py -v -rs -o "addopts=" \
+            2>&1 | tee /tmp/device_placement_test.log
+          device_placement_status=${PIPESTATUS[0]}
+          set -e
+          if [ "${device_placement_status}" -ne 0 ]; then
+            echo "::error::a device placement test failed"
+            exit "${device_placement_status}"
+          fi
+          if grep -qE "[0-9]+ skipped" /tmp/device_placement_test.log; then
+            echo "::error::a device placement test skipped in the CUDA job, which means the CUDA backend or a visible GPU is missing"
+            exit 1
+          fi
         fi
 
         # Build ExecuTorch with CUDA support
@@ -589,7 +613,7 @@ jobs:
           python -m pytest "${KERNEL_TESTS[@]}" -v -o "addopts=" --durations=20
         else
           IGNORES=()
-          for t in "${KERNEL_TESTS[@]}" "${QUANTIZED_GEMM_TESTS[@]}"; do
+          for t in "${KERNEL_TESTS[@]}" "${QUANTIZED_GEMM_TESTS[@]}" backends/cuda/tests/test_device_placement.py; do
             IGNORES+=("--ignore=${t}")
           done
           python -m pytest backends/cuda/tests backends/cuda/passes/tests -v -o "addopts=" --durations=20 \

.github/workflows/test-native.yml

@@ -0,0 +1,113 @@
+name: Test Native Engines
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/test-native.yml
+      - .github/workflows/_docker-image.yml
+      - .ci/docker/**
+      - .ci/scripts/test-native.sh
+      - .ci/scripts/test_native_llm_runner.py
+      - .gitmodules
+      - CMakeLists.txt
+      - install_requirements.py
+      - install_requirements.sh
+      - requirements-dev.txt
+      - requirements-examples.txt
+      - backends/native/**
+      - backends/vulkan/CMakeLists.txt
+      - backends/vulkan/cmake/**
+      - backends/vulkan/runtime/api/**
+      - backends/vulkan/runtime/vk_api/**
+      - backends/vulkan/runtime/graph/**
+      - backends/vulkan/runtime/utils/**
+      - backends/vulkan/runtime/gen_vulkan_spv.py
+      - backends/vulkan/third-party/Vulkan-Headers
+      - backends/vulkan/third-party/VulkanMemoryAllocator
+      - backends/vulkan/third-party/volk
+      - backends/vulkan_shared/CMakeLists.txt
+      - backends/vulkan_shared/runtime/**
+      - backends/xnnpack/third-party/FXdiv
+      - backends/xnnpack/third-party/cpuinfo
+      - backends/xnnpack/third-party/pthreadpool
+      - examples/models/llama/tokenizer/**
+      - extension/llm/tokenizers
+      - extension/llm/export/**
+      - exir/**
+      - runtime/**
+      - third-party/**
+      - tools/cmake/**
+      - build/**
+      - codegen/**
+      - schema/**
+  push:
+    branches:
+      - main
+      - release/*
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' && github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  docker-image:
+    name: Resolve CI docker image
+    uses: ./.github/workflows/_docker-image.yml
+
+  test-native:
+    name: native-${{ matrix.component }}
+    needs: docker-image
+    runs-on: mt-l-x86iavx512-8-64
+    container:
+      image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-gcc11-${{ needs.docker-image.outputs.ci-docker-hash }}
+    timeout-minutes: 60
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        component: [runtime, ggml, vulkan]
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Fix workspace permissions
+        run: sudo chown -R "$(id -u):$(id -g)" "${GITHUB_WORKSPACE}"
+
+      - name: Checkout ExecuTorch
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+          submodules: recursive
+          persist-credentials: false
+
+      - name: Build and test native ${{ matrix.component }}
+        env:
+          NATIVE_COMPONENT: ${{ matrix.component }}
+        run: |
+          set -exo pipefail
+          eval "$(/opt/conda/bin/conda shell.bash hook)"
+          CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+          conda activate "${CONDA_ENV}"
+
+          unset SCCACHE_BUCKET SCCACHE_S3_KEY_PREFIX
+          export SCCACHE_DIR=/tmp/sccache
+
+          packages=(libzip)
+          if [[ "${NATIVE_COMPONENT}" == "vulkan" ]]; then
+            packages+=(shaderc)
+          fi
+          conda create -y -p /tmp/native-deps -c conda-forge "${packages[@]}"
+          export CMAKE_PREFIX_PATH="/tmp/native-deps:${CMAKE_PREFIX_PATH:-}"
+          export PATH="/tmp/native-deps/bin:${PATH}"
+
+          if [[ "${NATIVE_COMPONENT}" == "ggml" ]]; then
+            ./install_requirements.sh --use-pt-pinned-commit
+            # The LLM runner test imports executorch from src/ without installing the
+            # wheel, so install its export-time dependencies directly.
+            pip install flatbuffers ruamel.yaml tabulate \
+              "$(grep -E '^transformers\b' requirements-examples.txt)"
+          fi
+
+          CMAKE_BUILD_PARALLEL_LEVEL=8 bash .ci/scripts/test-native.sh "${NATIVE_COMPONENT}"

CMakeLists.txt

@@ -980,7 +980,10 @@ if(EXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP)
 endif()
 
 if(EXECUTORCH_BUILD_EXTENSION_LLM)
-  if(EXECUTORCH_BUILD_EXTENSION_LLM_RUNNER)
+  if(EXECUTORCH_BUILD_EXTENSION_LLM_RUNNER
+     OR EXECUTORCH_BUILD_NATIVE_LLM_RUNNER
+  )
+    # Both LLM runners need tokenizers with regex lookahead support.
     set(SUPPORT_REGEX_LOOKAHEAD ON)
     # llama/runner/CMakeLists.txt builds a shared library libllama_runner.so
     # that transitively depends on tokenizers. Need to build tokenizers with
@@ -991,7 +994,9 @@ if(EXECUTORCH_BUILD_EXTENSION_LLM)
     set(CMAKE_POSITION_INDEPENDENT_CODE ON)
   endif()
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/llm/tokenizers)
-  if(EXECUTORCH_BUILD_EXTENSION_LLM_RUNNER)
+  if(EXECUTORCH_BUILD_EXTENSION_LLM_RUNNER
+     OR EXECUTORCH_BUILD_NATIVE_LLM_RUNNER
+  )
     set(CMAKE_POSITION_INDEPENDENT_CODE
         ${ORIGINAL_CMAKE_POSITION_INDEPENDENT_CODE_FLAG}
     )