diff --git a/.agents/vllm-backend.md b/.agents/vllm-backend.md index a2b9e614e466..a68bd506e8bb 100644 --- a/.agents/vllm-backend.md +++ b/.agents/vllm-backend.md @@ -21,6 +21,17 @@ options: - reasoning_parser:qwen3 ``` +## `Options[]` doubles as CLI-style engine flags + +Beyond the parser names above, `Options[]` carries `--` prefixed engine flags (`--enable-prefix-caching`, `--kv-cache-dtype:fp8_e5m2`). `apply_options_to_engine_args` in `backend/python/common/vllm_utils.py` maps them onto `AsyncEngineArgs` fields, and it must run **before** `AsyncLLMEngine.from_engine_args()` - applying them afterwards is a silent no-op, which is exactly what issue #11130 was. + +Things to keep straight when touching this: + +- Precedence is typed proto fields → `options:` → `engine_args:`. `applyEngineArgDefaults` in `core/config/hooks_vllm.go` therefore skips seeding a production default whose key the user already set as an option, otherwise the later `engine_args:` pass would silently override them. +- Only `--` prefixed entries are engine flags; `tool_parser:`/`reasoning_parser:` and friends keep their meaning. Parser lookups accept both spellings via `normalize_option_key`. +- Unknown or uncoercible flags warn and are skipped, unlike `engine_args:` which is strict - `Options[]` is a shared bag and knows entries this mapping doesn't. +- Field types come from the annotation's *base* (`Literal["auto","float16"]` is not a float). The helper's tests are stdlib-only: `make test-python-helpers`. + Auto-defaults for known model families live in `core/config/parser_defaults.json` and are applied: - at gallery import time by `core/gallery/importers/vllm.go` - at model load time by the `vllm` / `vllm-omni` backend hook in `core/config/hooks_vllm.go` diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 572764aea64c..44a8635174eb 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -66,3 +66,9 @@ jobs: node-version: '20' - name: run CI script tests run: make test-ci-scripts + + # The shared python backend helpers (Options[] parsing, engine-arg + # mapping, model reference resolution) are stdlib-only, so their tests + # ride along here instead of waiting on a multi-GB backend image build. + - name: run shared python backend helper tests + run: make test-python-helpers diff --git a/Makefile b/Makefile index 81b00d7c8d7b..e906863d866b 100644 --- a/Makefile +++ b/Makefile @@ -222,6 +222,14 @@ test-build-scripts: test-ci-scripts: @set -e; for t in scripts/lib/*_test.mjs; do echo "== $$t"; node --test "$$t"; done +## Runs the unit tests for the shared python backend helpers. These modules are +## pure stdlib on purpose so they run without any backend venv; the list is +## explicit because their siblings (model_identity_test) import grpc and the +## generated protobufs, which only exist inside a built backend. +PYTHON_HELPER_TESTS?=python_utils_test vllm_utils_test model_utils_test mlx_utils_test parent_watch_test +test-python-helpers: + cd backend/python/common && python3 -m unittest $(PYTHON_HELPER_TESTS) + ## Runs the core suite ($(TEST_PATHS)) with statement-coverage instrumentation ## and writes a merged profile to $(COVERAGE_PROFILE). Deliberately omits ## --fail-fast so a single failure doesn't truncate the coverage number, and diff --git a/backend/python/common/vllm_utils.py b/backend/python/common/vllm_utils.py index 9124645ac35d..60c38f5bd971 100644 --- a/backend/python/common/vllm_utils.py +++ b/backend/python/common/vllm_utils.py @@ -4,11 +4,208 @@ ``python_utils`` and are re-exported here for backwards compatibility with existing imports in both backends. """ +import dataclasses +import difflib +import json import sys from python_utils import messages_to_dicts, parse_options -__all__ = ["parse_options", "messages_to_dicts", "setup_parsers"] +__all__ = [ + "parse_options", + "messages_to_dicts", + "setup_parsers", + "apply_options_to_engine_args", + "normalize_option_key", +] + +# Options[] entries LocalAI itself acts on, in their normalized spelling. +_BACKEND_LEVEL_OPTIONS = {"tool_parser", "reasoning_parser"} + +_TRUTHY = {"true", "1", "yes", "on", "t", "y"} +_FALSY = {"false", "0", "no", "off", "f", "n"} + + +def normalize_option_key(key): + """Normalize an Options[] key to its engine/field spelling. + + Users copy flags straight out of vLLM's CLI docs (``--reasoning-parser``), + so accept that spelling everywhere LocalAI looks options up by name. + """ + return key.strip().lstrip("-").replace("-", "_") + + +_BASE_HINTS = { + "bool": "bool", + "int": "int", + "float": "float", + "str": "str", + "dict": "dict", + "mapping": "dict", +} + + +def _hint_from_annotation(annotation): + """Target type name for a field annotation, or None when it is not a scalar. + + ``dataclasses.fields()`` hands back either a real type, a ``typing`` + construct or - under PEP 563, which vLLM's arg_utils uses - the annotation + as a plain string, so work on the textual form. Match the *base* of the + annotation rather than searching it: ``Literal["auto", "float16"]`` (vLLM's + dtype) contains "float" but is not a float. + """ + if annotation is None: + return None + if isinstance(annotation, type): + text = annotation.__name__ + elif isinstance(annotation, str): + text = annotation + else: + text = str(annotation) + text = text.replace("typing.", "").strip() + while text.lower().startswith("optional[") and text.endswith("]"): + text = text[len("optional["):-1].strip() + if "|" in text: + parts = [p.strip() for p in text.split("|") if p.strip().lower() not in ("none", "nonetype")] + if len(parts) != 1: + return None + text = parts[0] + base = text.split("[", 1)[0].strip().lower() + return _BASE_HINTS.get(base) + + +def _hint_from_value(current): + """Target type name inferred from a field's current value.""" + if isinstance(current, bool): + return "bool" + if isinstance(current, dict): + return "dict" + if isinstance(current, float): + return "float" + if isinstance(current, int): + return "int" + if isinstance(current, str): + return "str" + return None + + +def _type_hint(annotation, current): + """Best-effort target type name for a dataclass field.""" + hint = _hint_from_annotation(annotation) + if hint is None: + hint = _hint_from_value(current) + return hint + + +def _coerce_option(raw, hint): + """Coerce a CLI-supplied string to the field's type. Raises ValueError.""" + if hint == "bool": + low = raw.lower() + if low in _TRUTHY: + return True + if low in _FALSY: + return False + raise ValueError(f"{raw!r} is not a boolean") + if hint == "int": + return int(raw) + if hint == "float": + return float(raw) + if hint == "dict": + return json.loads(raw) + if hint == "str": + return raw + + # Untyped (or union-typed) field: infer from the literal itself. + low = raw.lower() + if low in _TRUTHY: + return True + if low in _FALSY: + return False + for cast in (int, float): + try: + return cast(raw) + except ValueError: + pass + if raw.startswith("{") or raw.startswith("["): + return json.loads(raw) + return raw + + +def apply_options_to_engine_args(engine_args, options): + """Apply CLI-style ``--flag[:value]`` entries from Options[] to engine args. + + ``options:`` is a loose bag shared with backend-level settings + (``tool_parser:``, ``reasoning_parser:``, ``vad_only``, …), so only + ``--`` prefixed entries are treated as engine flags. Names are normalized + the way vLLM's own CLI does (``--enable-prefix-caching`` → + ``enable_prefix_caching``) and values are coerced to the target field's + type. Both ``--flag:value`` (LocalAI convention) and ``--flag=value`` + (vLLM CLI convention) are accepted; a bare ``--flag`` sets a boolean field. + + Unknown or uncoercible flags warn on stderr and are skipped instead of + failing the load - unlike ``engine_args:``, which is engine-only and + therefore strict, Options[] carries entries this function knows nothing + about. + + Returns a new dataclass instance via ``dataclasses.replace`` so the + engine's ``__post_init__`` re-runs, or the original when nothing applied. + """ + if not options: + return engine_args + + fields = {f.name: f for f in dataclasses.fields(type(engine_args))} + updates = {} + for opt in options: + opt = opt.strip() + if not opt.startswith("--"): + continue + body = opt[2:] + seps = [i for i in (body.find(":"), body.find("=")) if i != -1] + if seps: + cut = min(seps) + name, raw = body[:cut], body[cut + 1:].strip() + else: + name, raw = body, None + name = normalize_option_key(name) + + if name not in fields: + # LocalAI reads these itself; whether the engine dataclass carries + # them too varies by vLLM version, so never flag them as unknown. + if name in _BACKEND_LEVEL_OPTIONS: + continue + suggestion = difflib.get_close_matches(name, fields, n=1) + hint = f" did you mean {suggestion[0]!r}?" if suggestion else "" + print( + f"[vllm_utils] unknown engine option {opt!r} (field {name!r}), skipping.{hint}", + file=sys.stderr, + ) + continue + + target = _type_hint(fields[name].type, getattr(engine_args, name, None)) + if raw is None: + if target != "bool": + print( + f"[vllm_utils] engine option {opt!r} needs a value " + f"(field {name!r} is not a flag), skipping", + file=sys.stderr, + ) + continue + updates[name] = True + continue + + try: + updates[name] = _coerce_option(raw, target) + except (ValueError, TypeError) as err: + print( + f"[vllm_utils] cannot apply engine option {opt!r} to field " + f"{name!r}: {err}, skipping", + file=sys.stderr, + ) + + if not updates: + return engine_args + print(f"[vllm_utils] engine options from Options[]: {updates}", file=sys.stderr) + return dataclasses.replace(engine_args, **updates) def setup_parsers(opts): diff --git a/backend/python/common/vllm_utils_test.py b/backend/python/common/vllm_utils_test.py new file mode 100644 index 000000000000..b3bbcbb5b465 --- /dev/null +++ b/backend/python/common/vllm_utils_test.py @@ -0,0 +1,160 @@ +"""Unit tests for the shared vLLM backend helpers (vllm_utils.py). + +Run standalone (Python standard library only, no backend venv needed): + cd backend/python/common && python3 -m unittest vllm_utils_test + +``vllm_utils`` imports vLLM lazily (inside functions), so the module is +importable without the vLLM wheel. ``AsyncEngineArgs`` is stood in for by a +local dataclass that mirrors the field shapes that matter: plain scalars, +``Optional[...]`` scalars (vLLM's tri-state flags) and dict-valued fields. +""" + +import contextlib +import dataclasses +import io +import unittest +from typing import Dict, Literal, Optional, Union + +from vllm_utils import apply_options_to_engine_args, normalize_option_key + + +@dataclasses.dataclass +class FakeEngineArgs: + model: str = "" + quantization: Optional[str] = None + kv_cache_dtype: str = "auto" + enable_prefix_caching: Optional[bool] = None + enforce_eager: bool = False + max_model_len: Optional[int] = None + tensor_parallel_size: int = 1 + gpu_memory_utilization: float = 0.9 + limit_mm_per_prompt: Optional[Dict[str, int]] = None + # vLLM types dtype as a Literal of strings, several of which contain the + # word "float" - a naive match reads that as a float field. + dtype: Literal["auto", "float16", "bfloat16"] = "auto" + seed: Union[int, str, None] = None + + +# vLLM's arg_utils is annotated under PEP 563, where dataclasses.fields() hands +# back annotations as plain strings rather than types. +StringAnnotatedEngineArgs = dataclasses.make_dataclass( + "StringAnnotatedEngineArgs", + [ + ("max_model_len", "int | None", dataclasses.field(default=None)), + ("enable_prefix_caching", "Optional[bool]", dataclasses.field(default=None)), + ("kv_cache_dtype", "str", dataclasses.field(default="auto")), + ], +) + + +def _apply(options, **overrides): + """Apply options to a fresh FakeEngineArgs, returning (result, stderr).""" + err = io.StringIO() + with contextlib.redirect_stderr(err): + out = apply_options_to_engine_args(FakeEngineArgs(**overrides), options) + return out, err.getvalue() + + +class TestApplyOptionsToEngineArgs(unittest.TestCase): + def test_string_option_is_applied(self): + out, _ = _apply(["--quantization:gptq_marlin"]) + self.assertEqual(out.quantization, "gptq_marlin") + + def test_valueless_flag_enables_boolean_field(self): + out, _ = _apply(["--enable-prefix-caching"]) + self.assertIs(out.enable_prefix_caching, True) + + def test_boolean_field_accepts_explicit_false(self): + out, _ = _apply(["--enable-prefix-caching:false"], enable_prefix_caching=True) + self.assertIs(out.enable_prefix_caching, False) + + def test_integer_field_is_coerced(self): + out, _ = _apply(["--max-model-len:4096", "--tensor-parallel-size:2"]) + self.assertEqual(out.max_model_len, 4096) + self.assertEqual(out.tensor_parallel_size, 2) + + def test_float_field_is_coerced(self): + out, _ = _apply(["--gpu-memory-utilization:0.85"]) + self.assertAlmostEqual(out.gpu_memory_utilization, 0.85) + + def test_equals_separator_is_accepted(self): + out, _ = _apply(["--kv-cache-dtype=fp8_e5m2"]) + self.assertEqual(out.kv_cache_dtype, "fp8_e5m2") + + def test_dict_field_is_parsed_as_json(self): + out, _ = _apply(['--limit-mm-per-prompt:{"image": 4}']) + self.assertEqual(out.limit_mm_per_prompt, {"image": 4}) + + def test_non_flag_options_are_left_alone(self): + out, err = _apply(["tool_parser:hermes", "reasoning_parser:qwen3", "vad_only"]) + self.assertEqual(out, FakeEngineArgs()) + self.assertEqual(err, "") + + def test_unknown_flag_warns_and_is_skipped(self): + out, err = _apply(["--not-a-real-flag:1", "--quantization:awq"]) + self.assertEqual(out.quantization, "awq") + self.assertIn("not_a_real_flag", err) + self.assertIn("unknown", err.lower()) + + def test_parser_flags_are_not_reported_as_unknown(self): + # LocalAI consumes these itself; whether the engine dataclass also has + # the field depends on the vLLM version, and warning about them would + # send users chasing a non-problem. + out, err = _apply(["--tool-parser:hermes", "--reasoning-parser:qwen3"]) + self.assertEqual(out, FakeEngineArgs()) + self.assertEqual(err, "") + + def test_unknown_flag_hints_at_the_closest_field(self): + _, err = _apply(["--max-model-length:4096"]) + self.assertIn("max_model_len", err) + + def test_uncoercible_value_warns_and_is_skipped(self): + out, err = _apply(["--max-model-len:lots"]) + self.assertIsNone(out.max_model_len) + self.assertIn("max_model_len", err) + + def test_valueless_flag_on_non_boolean_field_warns_and_is_skipped(self): + out, err = _apply(["--quantization"]) + self.assertIsNone(out.quantization) + self.assertIn("quantization", err) + + def test_empty_options_returns_the_same_engine_args(self): + original = FakeEngineArgs(model="m") + self.assertIs(apply_options_to_engine_args(original, []), original) + + +class TestFieldTypeInference(unittest.TestCase): + def test_literal_typed_field_keeps_its_string_value(self): + out, err = _apply(["--dtype:bfloat16"]) + self.assertEqual(out.dtype, "bfloat16") + self.assertEqual(err.count("skipping"), 0) + + def test_ambiguous_union_falls_back_to_literal_inference(self): + out, _ = _apply(["--seed:42"]) + self.assertEqual(out.seed, 42) + + def test_string_annotations_are_understood(self): + err = io.StringIO() + with contextlib.redirect_stderr(err): + out = apply_options_to_engine_args( + StringAnnotatedEngineArgs(), + ["--max-model-len:4096", "--enable-prefix-caching", "--kv-cache-dtype:fp8_e5m2"], + ) + self.assertEqual(out.max_model_len, 4096) + self.assertIs(out.enable_prefix_caching, True) + self.assertEqual(out.kv_cache_dtype, "fp8_e5m2") + + +class TestNormalizeOptionKey(unittest.TestCase): + def test_cli_flag_becomes_a_field_name(self): + self.assertEqual(normalize_option_key("--reasoning-parser"), "reasoning_parser") + + def test_plain_key_is_untouched(self): + self.assertEqual(normalize_option_key("tool_parser"), "tool_parser") + + def test_surrounding_whitespace_is_stripped(self): + self.assertEqual(normalize_option_key(" --tool-parser "), "tool_parser") + + +if __name__ == "__main__": + unittest.main() diff --git a/backend/python/vllm/backend.py b/backend/python/vllm/backend.py index 8a42e7bcbf4d..a5a85041bc3a 100644 --- a/backend/python/vllm/backend.py +++ b/backend/python/vllm/backend.py @@ -22,6 +22,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common')) from grpc_auth import get_auth_interceptors from model_utils import resolve_model_reference +from vllm_utils import apply_options_to_engine_args, normalize_option_key from vllm.engine.arg_utils import AsyncEngineArgs from vllm.engine.async_llm_engine import AsyncLLMEngine @@ -101,13 +102,18 @@ def generate(self,prompt, max_new_tokens): return decoded_text def _parse_options(self, options_list): - """Parse Options[] key:value string list into a dict.""" + """Parse Options[] key:value string list into a dict. + + Keys are normalized to their field spelling so the CLI form users copy + from vLLM's docs (``--reasoning-parser:qwen3``) selects the same parser + as LocalAI's own (``reasoning_parser:qwen3``). + """ opts = {} for opt in options_list: if ":" not in opt: continue key, value = opt.split(":", 1) - opts[key.strip()] = value.strip() + opts[normalize_option_key(key)] = value.strip() return opts def _apply_engine_args(self, engine_args, engine_args_json): @@ -229,6 +235,12 @@ async def LoadModel(self, request, context): "audio": max(request.LimitAudioPerPrompt, 1) } + # CLI-style flags in options: (--quantization:gptq_marlin, + # --enable-prefix-caching, ...) land on the engine args too - they must + # be applied *before* the engine is created or they do nothing. + # engine_args: is applied after this, so it stays the last word. + engine_args = apply_options_to_engine_args(engine_args, request.Options) + # engine_args from YAML overrides typed fields above so operators can # tune anything the AsyncEngineArgs dataclass exposes without waiting # on protobuf changes. diff --git a/core/config/hooks_test.go b/core/config/hooks_test.go index 4a6e31d6cf0b..b69bc6989bb2 100644 --- a/core/config/hooks_test.go +++ b/core/config/hooks_test.go @@ -198,6 +198,31 @@ var _ = Describe("Backend hooks and parser defaults", func() { // chunked_prefill is still seeded since user didn't set it Expect(cfg.EngineArgs["enable_chunked_prefill"]).To(Equal(true)) }) + + // The backend applies options: before engine_args:, so seeding a default + // for a key the user already set through a CLI-style option would silently + // win over it. https://github.com/mudler/LocalAI/issues/11130 + It("does not seed a default the user set through options", func() { + cfg := &ModelConfig{ + Backend: "vllm", + Options: []string{"--enable-prefix-caching:false", "--enable-chunked-prefill"}, + } + cfg.SetDefaults() + + Expect(cfg.EngineArgs).NotTo(HaveKey("enable_prefix_caching")) + Expect(cfg.EngineArgs).NotTo(HaveKey("enable_chunked_prefill")) + }) + + It("still seeds defaults when options carry unrelated entries", func() { + cfg := &ModelConfig{ + Backend: "vllm", + Options: []string{"tool_parser:hermes", "--quantization:gptq_marlin"}, + } + cfg.SetDefaults() + + Expect(cfg.EngineArgs["enable_prefix_caching"]).To(Equal(true)) + Expect(cfg.EngineArgs["enable_chunked_prefill"]).To(Equal(true)) + }) }) Context("llamaCppDefaults GGUF guessing", func() { diff --git a/core/config/hooks_vllm.go b/core/config/hooks_vllm.go index ffdd1a52a2aa..67408511bde2 100644 --- a/core/config/hooks_vllm.go +++ b/core/config/hooks_vllm.go @@ -59,19 +59,48 @@ func vllmDefaults(cfg *ModelConfig, modelPath string) { } // applyEngineArgDefaults seeds production-friendly engine_args without overwriting -// anything the user already set. +// anything the user already set, in engine_args or as a CLI-style option. func applyEngineArgDefaults(cfg *ModelConfig) { if cfg.EngineArgs == nil { cfg.EngineArgs = map[string]any{} } + fromOptions := engineOptionKeys(cfg.Options) for k, v := range productionEngineArgsDefaults { if _, set := cfg.EngineArgs[k]; set { continue } + // The backend applies options: before engine_args:, so seeding a key + // the user wrote as an option would silently override it. + if _, set := fromOptions[k]; set { + continue + } cfg.EngineArgs[k] = v } } +// engineOptionKeys returns the engine-arg field names carried by CLI-style +// options (`--enable-prefix-caching:false` -> `enable_prefix_caching`). Only +// `--` prefixed entries are engine flags; the rest of Options[] is +// backend-level (tool_parser:, reasoning_parser:, ...). +func engineOptionKeys(options []string) map[string]struct{} { + keys := map[string]struct{}{} + for _, opt := range options { + opt = strings.TrimSpace(opt) + if !strings.HasPrefix(opt, "--") { + continue + } + name := opt + if i := strings.IndexAny(opt, ":="); i != -1 { + name = opt[:i] + } + name = strings.ReplaceAll(strings.TrimLeft(name, "-"), "-", "_") + if name != "" { + keys[name] = struct{}{} + } + } + return keys +} + func applyParserDefaults(cfg *ModelConfig) { hasToolParser := false hasReasoningParser := false diff --git a/docs/content/features/text-generation.md b/docs/content/features/text-generation.md index f7cc155a80f5..9564ab0ca8e4 100644 --- a/docs/content/features/text-generation.md +++ b/docs/content/features/text-generation.md @@ -766,6 +766,35 @@ engine_args: attention_backend: TRITON_ATTN ``` +#### CLI-style engine flags in `options` + +Engine arguments can also be written in `options:` with vLLM's own CLI +spelling, which is handy when copying a command line out of vLLM's +documentation: + +```yaml +options: + - --quantization:gptq_marlin + - --enable-prefix-caching + - --kv-cache-dtype:fp8_e5m2 + - --reasoning-parser:qwen3 +``` + +Rules: + +- Only `--` prefixed entries are treated as engine flags; everything + else in `options:` (`tool_parser:`, `reasoning_parser:`, …) keeps its + existing meaning. Both `--flag:value` and `--flag=value` are accepted. +- Dashes become underscores (`--enable-prefix-caching` → + `enable_prefix_caching`) and the value is coerced to the target + field's type. A bare `--flag` sets a boolean field to `true`. +- Unknown or uncoercible flags are logged on the backend's stderr and + skipped rather than failing the load. `engine_args:` stays strict and + is applied last, so it wins on conflict. + +`engine_args:` remains the form to use for anything structured (nested +configs, maps); `options:` is a convenience for flat flags. + #### Multi-node data parallelism `engine_args.data_parallel_size > 1` combined with the