From e98e318cd8907da95fdd66fda96561984d84acaf Mon Sep 17 00:00:00 2001 From: Jake Stevens Date: Tue, 27 Oct 2026 13:55:31 -0700 Subject: [PATCH] Add CMake targets for HF LLM runners (#23899) Summary: Add `EXECUTORCH_BUILD_NATIVE_LLM_RUNNER` to build and install `run_hf_llm_vulkan` and `run_hf_llm_ggml` for the enabled engines. Share gflags setup with the vision runner, resolve tokenizers with regex lookahead support, and add `native_llm_runner_utils_test`. Document building and running the GGML CPU LLM runner. Authored with assistance from Codex. Reviewed By: digantdesai Differential Revision: D125479665 --- .ci/scripts/test-native.sh | 20 +++- .ci/scripts/test_native_llm_runner.py | 109 ++++++++++++++++++ .github/workflows/test-native.yml | 17 +++ CMakeLists.txt | 9 +- backends/native/CMakeLists.txt | 2 +- backends/native/README.md | 41 ++++++- backends/native/examples/CMakeLists.txt | 16 ++- .../examples/hf_llm/runtime/CMakeLists.txt | 62 ++++++++++ .../native/examples/vision/CMakeLists.txt | 11 +- backends/native/test/CMakeLists.txt | 19 +++ tools/cmake/preset/default.cmake | 5 + 11 files changed, 289 insertions(+), 22 deletions(-) create mode 100644 .ci/scripts/test_native_llm_runner.py create mode 100644 backends/native/examples/hf_llm/runtime/CMakeLists.txt diff --git a/.ci/scripts/test-native.sh b/.ci/scripts/test-native.sh index ebd4277d4bc..2f08fc6dabf 100644 --- a/.ci/scripts/test-native.sh +++ b/.ci/scripts/test-native.sh @@ -20,17 +20,22 @@ case "${component}" in test_regex='^native_runtime_test$' ;; ggml) - cmake_args+=(-DEXECUTORCH_BUILD_NATIVE_GGML=ON -DGGML_NATIVE=OFF) - targets+=(native_ggml_engine_test) - test_regex='^native_ggml_engine_test$' + cmake_args+=( + -DEXECUTORCH_BUILD_NATIVE_GGML=ON + -DEXECUTORCH_BUILD_NATIVE_LLM_RUNNER=ON + -DGGML_NATIVE=OFF + ) + targets+=(run_hf_llm_ggml native_ggml_engine_test native_llm_runner_utils_test) + test_regex='^native_(ggml_engine|llm_runner_utils)_test$' ;; vulkan) cmake_args+=( -DEXECUTORCH_BUILD_NATIVE_VULKAN=ON -DEXECUTORCH_BUILD_NATIVE_VISION_RUNNER=ON + -DEXECUTORCH_BUILD_NATIVE_LLM_RUNNER=ON -DEXECUTORCH_VULKAN_SHADER_COMPILE_NTHREADS="${build_jobs}" ) - targets+=(native_vulkan_engine run_classification native_vulkan_passes_test native_runner_utils_test) + targets+=(native_vulkan_engine run_classification run_hf_llm_vulkan native_vulkan_passes_test native_runner_utils_test) test_regex='^native_(vulkan_passes|runner_utils)_test$' ;; *) @@ -53,3 +58,10 @@ cmake --build "${build_dir}" \ --target "${targets[@]}" ctest --test-dir "${build_dir}/backends/native/test" \ -R "${test_regex}" --output-on-failure --no-tests=error --timeout 120 + +if [[ "${component}" == "ggml" ]]; then + PYTHONPATH="${PWD}/src:${PYTHONPATH:-}" \ + FLATC_EXECUTABLE="${FLATC_EXECUTABLE:-${PWD}/${build_dir}/third-party/flatc_ep/bin/flatc}" \ + python .ci/scripts/test_native_llm_runner.py \ + --runner "${build_dir}/backends/native/examples/hf_llm/runtime/run_hf_llm_ggml" +fi diff --git a/.ci/scripts/test_native_llm_runner.py b/.ci/scripts/test_native_llm_runner.py new file mode 100644 index 00000000000..9fe9eadf638 --- /dev/null +++ b/.ci/scripts/test_native_llm_runner.py @@ -0,0 +1,109 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Compare GGML LLM runner prefill and decoding with a tiny eager HF Llama.""" + +import argparse +import base64 +import re +import subprocess +import tempfile +from pathlib import Path + +import torch +from executorch.backends.native.examples.hf_llm.export_hf_llm import LastTokenLM +from executorch.exir.native import to_native +from transformers import LlamaConfig, LlamaForCausalLM + + +@torch.no_grad() +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--runner", required=True, type=Path) + args = parser.parse_args() + + torch.manual_seed(0) + torch.set_num_threads(4) + max_seq_length = 48 + max_new_tokens = 8 + config = LlamaConfig( + vocab_size=512, + hidden_size=32, + intermediate_size=64, + num_hidden_layers=2, + num_attention_heads=4, + num_key_value_heads=2, + max_position_embeddings=max_seq_length, + bos_token_id=256, + eos_token_id=257, + attn_implementation="sdpa", + ) + model = LlamaForCausalLM(config).eval() + model.generation_config.use_cache = True + model.generation_config.cache_implementation = "static" + wrapper = LastTokenLM(model, max_seq_length).eval() + seq = torch.export.Dim("seq", min=1, max=max_seq_length - 1) + exported = torch.export.export( + wrapper, + (torch.zeros((1, 6), dtype=torch.long), torch.arange(6)), + dynamic_shapes={"input_ids": {1: seq}, "cache_position": {0: seq}}, + strict=True, + ) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + package = root / "llama.ptn" + to_native(exported).save(str(package)) + tokenizer = root / "tokenizer.model" + # The byte vocabulary puts Llama's BOS at 256, followed by 255 specials. + tokenizer.write_text( + "".join( + f"{base64.b64encode(bytes([i])).decode()} {i}\n" for i in range(256) + ) + ) + prompt = "abc" + unreachable_eos_id = config.vocab_size + tokens = [config.bos_token_id, *prompt.encode()] + expected = [] + # Recompute the full prefix without a cache for an independent reference. + for _ in range(max_new_tokens): + logits = model(torch.tensor([tokens]), use_cache=False).logits + token = int(logits[0, -1].argmax()) + expected.append(token) + tokens.append(token) + + result = subprocess.run( + [ + str(args.runner.resolve()), + "--device=cpu", + "--cpu_threads=4", + f"--package={package}", + f"--tokenizer={tokenizer}", + f"--prompt={prompt}", + f"--eos_ids={unreachable_eos_id}", + f"--max_new_tokens={max_new_tokens}", + f"--max_seq_length={max_seq_length}", + "--print_token_ids", + ], + capture_output=True, + text=True, + errors="replace", + timeout=120, + ) + print(result.stdout, end="") + print(result.stderr, end="") + result.check_returncode() + match = re.search(r"^token_ids: ([0-9,]+)$", result.stdout, re.MULTILINE) + if match is None: + raise AssertionError("Runner did not report generated token IDs") + actual = [int(token) for token in match[1].split(",")] + if actual != expected: + raise AssertionError(f"Generated tokens {actual} != eager {expected}") + print(f"GGML HF LLM runner matched all {max_new_tokens} eager token IDs") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/test-native.yml b/.github/workflows/test-native.yml index 6dd7e5db718..c5e0f0b7c76 100644 --- a/.github/workflows/test-native.yml +++ b/.github/workflows/test-native.yml @@ -7,8 +7,13 @@ on: - .github/workflows/_docker-image.yml - .ci/docker/** - .ci/scripts/test-native.sh + - .ci/scripts/test_native_llm_runner.py - .gitmodules - CMakeLists.txt + - install_requirements.py + - install_requirements.sh + - requirements-dev.txt + - requirements-examples.txt - backends/native/** - backends/vulkan/CMakeLists.txt - backends/vulkan/cmake/** @@ -25,6 +30,10 @@ on: - backends/xnnpack/third-party/FXdiv - backends/xnnpack/third-party/cpuinfo - backends/xnnpack/third-party/pthreadpool + - examples/models/llama/tokenizer/** + - extension/llm/tokenizers + - extension/llm/export/** + - exir/** - runtime/** - third-party/** - tools/cmake/** @@ -93,4 +102,12 @@ jobs: export CMAKE_PREFIX_PATH="/tmp/native-deps:${CMAKE_PREFIX_PATH:-}" export PATH="/tmp/native-deps/bin:${PATH}" + if [[ "${NATIVE_COMPONENT}" == "ggml" ]]; then + ./install_requirements.sh --use-pt-pinned-commit + # The LLM runner test imports executorch from src/ without installing the + # wheel, so install its export-time dependencies directly. + pip install flatbuffers ruamel.yaml tabulate \ + "$(grep -E '^transformers\b' requirements-examples.txt)" + fi + CMAKE_BUILD_PARALLEL_LEVEL=8 bash .ci/scripts/test-native.sh "${NATIVE_COMPONENT}" diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ae92a457b5..3768d04a083 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -980,7 +980,10 @@ if(EXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP) endif() if(EXECUTORCH_BUILD_EXTENSION_LLM) - if(EXECUTORCH_BUILD_EXTENSION_LLM_RUNNER) + if(EXECUTORCH_BUILD_EXTENSION_LLM_RUNNER + OR EXECUTORCH_BUILD_NATIVE_LLM_RUNNER + ) + # Both LLM runners need tokenizers with regex lookahead support. set(SUPPORT_REGEX_LOOKAHEAD ON) # llama/runner/CMakeLists.txt builds a shared library libllama_runner.so # that transitively depends on tokenizers. Need to build tokenizers with @@ -991,7 +994,9 @@ if(EXECUTORCH_BUILD_EXTENSION_LLM) set(CMAKE_POSITION_INDEPENDENT_CODE ON) endif() add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/llm/tokenizers) - if(EXECUTORCH_BUILD_EXTENSION_LLM_RUNNER) + if(EXECUTORCH_BUILD_EXTENSION_LLM_RUNNER + OR EXECUTORCH_BUILD_NATIVE_LLM_RUNNER + ) set(CMAKE_POSITION_INDEPENDENT_CODE ${ORIGINAL_CMAKE_POSITION_INDEPENDENT_CODE_FLAG} ) diff --git a/backends/native/CMakeLists.txt b/backends/native/CMakeLists.txt index e111c8f5453..2b4b8ec1a7c 100644 --- a/backends/native/CMakeLists.txt +++ b/backends/native/CMakeLists.txt @@ -36,7 +36,7 @@ install( PATTERN "ggml" EXCLUDE ) -if(EXECUTORCH_BUILD_NATIVE_VISION_RUNNER) +if(EXECUTORCH_BUILD_NATIVE_VISION_RUNNER OR EXECUTORCH_BUILD_NATIVE_LLM_RUNNER) add_subdirectory(examples) endif() diff --git a/backends/native/README.md b/backends/native/README.md index cb1f5af1eed..6ef90566b0e 100644 --- a/backends/native/README.md +++ b/backends/native/README.md @@ -110,16 +110,21 @@ target_link_libraries(my_app PRIVATE native_api native_vulkan_engine) With `EXECUTORCH_BUILD_TESTS=ON`, `native_runtime_test` covers the shared runtime and API. `native_vulkan_passes_test` covers the Vulkan graph rewrites without -requiring a GPU. When the vision runner below is enabled, -`native_runner_utils_test` covers its shared runner utilities. Run these with +requiring a GPU. When either runner below is enabled, +`native_runner_utils_test` covers their shared runner utilities. Run these with `ctest --test-dir cmake-out -R native_`. The `Test Native Engines` CI workflow has separate Linux jobs for the shared runtime, GGML, and Vulkan, with each test suite run once. To reproduce with dependencies installed, run `bash .ci/scripts/test-native.sh runtime`, `bash .ci/scripts/test-native.sh ggml`, or -`bash .ci/scripts/test-native.sh vulkan` from the OSS checkout. The Vulkan job -also builds `run_classification`; its tests do not require a GPU. +`bash .ci/scripts/test-native.sh vulkan` from the OSS checkout. The GGML job +also runs `native_llm_runner_utils_test` and executes `run_hf_llm_ggml` on a tiny +seeded Hugging Face Llama with a generated byte tokenizer. Its eight generated +token IDs must match eager PyTorch, covering prefill and KV-cache decoding without +downloading a model. The GGML job requires the Python export dependencies +installed by the `Test Native Engines` workflow. The Vulkan job builds +`run_classification` and `run_hf_llm_vulkan`; its tests do not require a GPU. ### GGML engine @@ -177,3 +182,31 @@ Export the `.ptn`, input, reference logits, and labels with [`export.py`](../../examples/native/vision/export.py). The runner prints the top-5 logits and, when `--expected` is given, exits nonzero if the logits differ from the reference beyond `--rtol`/`--atol` (both default to `1e-3`). + +## Native LLM runner + +`EXECUTORCH_BUILD_NATIVE_LLM_RUNNER=ON` builds `run_hf_llm_vulkan` with Native +Vulkan and `run_hf_llm_ggml` with Native GGML. It requires gflags and +`tokenizers::tokenizers` (the vendored tokenizer is built automatically in an +OSS checkout; otherwise provide an installed tokenizers package with regex +lookahead support). It accepts Llama 3 TikToken tokenizer files. + +To build the text runner for GGML CPU, initialize the llama.cpp submodule as above: + +```sh +cmake -S . -B cmake-out -DCMAKE_BUILD_TYPE=Release \ + -DEXECUTORCH_BUILD_NATIVE=ON -DEXECUTORCH_BUILD_NATIVE_GGML=ON \ + -DEXECUTORCH_BUILD_NATIVE_LLM_RUNNER=ON +cmake --build cmake-out --target run_hf_llm_ggml +cmake --install cmake-out --prefix install +install/bin/run_hf_llm_ggml --device cpu --cpu_threads 4 \ + --package llama.ptn --tokenizer tokenizer.model \ + --prompt "The capital of France is" --max_new_tokens 32 +``` + +The GGML runner accepts `--device cpu|gpu`, `--device_index`, `--cpu_threads`, +and `--max_cached_graphs` (default 1, 0 rebuilds each call). For GPU execution, +enable the desired `GGML_*` backend when configuring ExecuTorch, as above. + +With `EXECUTORCH_BUILD_TESTS=ON`, `native_llm_runner_utils_test` exercises the +LLM generation utilities. diff --git a/backends/native/examples/CMakeLists.txt b/backends/native/examples/CMakeLists.txt index 81b683a7d05..2f6235040eb 100644 --- a/backends/native/examples/CMakeLists.txt +++ b/backends/native/examples/CMakeLists.txt @@ -4,5 +4,19 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. +if(NOT TARGET gflags AND NOT TARGET gflags::gflags) + find_package(gflags REQUIRED) +endif() +if(TARGET gflags::gflags) + set(_native_runner_gflags gflags::gflags) +else() + set(_native_runner_gflags gflags) +endif() + add_subdirectory(runner_utils) -add_subdirectory(vision) +if(EXECUTORCH_BUILD_NATIVE_VISION_RUNNER) + add_subdirectory(vision) +endif() +if(EXECUTORCH_BUILD_NATIVE_LLM_RUNNER) + add_subdirectory(hf_llm/runtime) +endif() diff --git a/backends/native/examples/hf_llm/runtime/CMakeLists.txt b/backends/native/examples/hf_llm/runtime/CMakeLists.txt new file mode 100644 index 00000000000..5e3db657663 --- /dev/null +++ b/backends/native/examples/hf_llm/runtime/CMakeLists.txt @@ -0,0 +1,62 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +if(NOT TARGET tokenizers::tokenizers) + if(EXISTS "${EXECUTORCH_ROOT}/extension/llm/tokenizers/CMakeLists.txt") + set(SUPPORT_REGEX_LOOKAHEAD ON) + add_subdirectory( + ${EXECUTORCH_ROOT}/extension/llm/tokenizers + ${CMAKE_CURRENT_BINARY_DIR}/tokenizers + ) + else() + find_package(tokenizers CONFIG REQUIRED) + endif() +endif() + +add_library(native_llm_runner_utils STATIC LlmRunnerUtils.cpp) +target_compile_features(native_llm_runner_utils PUBLIC cxx_std_20) +target_include_directories( + native_llm_runner_utils PUBLIC $ +) +add_library( + native_llm_runner STATIC + LlmRunner.cpp + ${EXECUTORCH_ROOT}/examples/models/llama/tokenizer/llama_tiktoken.cpp +) +target_link_libraries( + native_llm_runner + PUBLIC native_engine + PRIVATE ${_native_runner_gflags} native_llm_runner_utils native_package + native_runner_utils native_runtime tokenizers::tokenizers +) +foreach(target native_llm_runner native_llm_runner_utils) + target_compile_options( + ${target} PRIVATE $<$:/EHsc> + $<$>:-fexceptions> + ) +endforeach() + +set(_llm_runners) +if(EXECUTORCH_BUILD_NATIVE_VULKAN) + add_executable(run_hf_llm_vulkan run_hf_llm_vulkan.cpp) + target_link_libraries(run_hf_llm_vulkan PRIVATE native_vulkan_engine) + list(APPEND _llm_runners run_hf_llm_vulkan) +endif() +if(EXECUTORCH_BUILD_NATIVE_GGML) + add_executable(run_hf_llm_ggml run_hf_llm_ggml.cpp) + target_link_libraries(run_hf_llm_ggml PRIVATE native_ggml_engine) + list(APPEND _llm_runners run_hf_llm_ggml) +endif() +foreach(target IN LISTS _llm_runners) + target_link_libraries( + ${target} PRIVATE native_llm_runner ${_native_runner_gflags} + ) + target_compile_options( + ${target} PRIVATE $<$:/EHsc> + $<$>:-fexceptions> + ) + install(TARGETS ${target} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +endforeach() diff --git a/backends/native/examples/vision/CMakeLists.txt b/backends/native/examples/vision/CMakeLists.txt index af7ea765224..76bfdb282f3 100644 --- a/backends/native/examples/vision/CMakeLists.txt +++ b/backends/native/examples/vision/CMakeLists.txt @@ -4,19 +4,10 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -if(NOT TARGET gflags AND NOT TARGET gflags::gflags) - find_package(gflags REQUIRED) -endif() -if(TARGET gflags::gflags) - set(_native_vision_gflags gflags::gflags) -else() - set(_native_vision_gflags gflags) -endif() - add_executable(run_classification run_classification.cpp) target_link_libraries( run_classification - PRIVATE ${_native_vision_gflags} native_engine native_package + PRIVATE ${_native_runner_gflags} native_engine native_package native_runner_utils native_runtime native_vulkan_engine ) target_compile_options( diff --git a/backends/native/test/CMakeLists.txt b/backends/native/test/CMakeLists.txt index 68b9b119f94..fb321e8736e 100644 --- a/backends/native/test/CMakeLists.txt +++ b/backends/native/test/CMakeLists.txt @@ -79,3 +79,22 @@ if(TARGET native_runner_utils) $<$>:-fexceptions> ) endif() + +if(TARGET native_llm_runner_utils) + add_executable( + native_llm_runner_utils_test + ../examples/hf_llm/runtime/LlmRunnerUtilsTest.cpp + ) + target_link_libraries( + native_llm_runner_utils_test PRIVATE native_llm_runner_utils + GTest::gtest_main + ) + add_test(NAME native_llm_runner_utils_test + COMMAND native_llm_runner_utils_test + ) + target_compile_options( + native_llm_runner_utils_test + PRIVATE $<$:/EHsc> + $<$>:-fexceptions> + ) +endif() diff --git a/tools/cmake/preset/default.cmake b/tools/cmake/preset/default.cmake index 039c5ce3712..acd332f0bda 100644 --- a/tools/cmake/preset/default.cmake +++ b/tools/cmake/preset/default.cmake @@ -195,9 +195,14 @@ cmake_dependent_option( "Build the native vision classification runner (run_classification)" OFF "EXECUTORCH_BUILD_NATIVE_VULKAN" OFF ) +cmake_dependent_option( + EXECUTORCH_BUILD_NATIVE_LLM_RUNNER "Build the native LLM runners" OFF + "EXECUTORCH_BUILD_NATIVE_VULKAN OR EXECUTORCH_BUILD_NATIVE_GGML" OFF +) announce_configured_options(EXECUTORCH_BUILD_NATIVE_VULKAN) announce_configured_options(EXECUTORCH_BUILD_NATIVE_GGML) announce_configured_options(EXECUTORCH_BUILD_NATIVE_VISION_RUNNER) +announce_configured_options(EXECUTORCH_BUILD_NATIVE_LLM_RUNNER) define_overridable_option( EXECUTORCH_BUILD_WEBGPU "Build the WebGPU backend" BOOL OFF )