From 0339f43153636823e146db5749ca85195a8d8f14 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Fri, 3 Jul 2026 08:01:31 +0800 Subject: [PATCH 1/2] fix(ai-cache): bypass caching for prompts carrying non-text content Since #13634 the protocol get_messages() flattens structured message content to plain text, dropping image/audio/other typed parts. ai-cache consumed that flattened view, so a text+image prompt became indistinguishable from a text-only one: they collided on the same L1 exact key and on the same L2 semantic cell, letting a multimodal request be wrongly served a text-only cached response (X-AI-Cache-Status: HIT). Detect non-text content from the RAW request body (before flattening) and bypass the cache entirely for such a request -- read nothing, write nothing, report a MISS -- so it always proceeds to the upstream. This restores the multimodal-bypass invariant guarded by t/plugin/ai-cache-semantic.t TEST 62-64. --- apisix/plugins/ai-cache.lua | 11 ++++++++++ apisix/plugins/ai-cache/semantic.lua | 31 +++++++++++++++++++++------- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/apisix/plugins/ai-cache.lua b/apisix/plugins/ai-cache.lua index 895ccc7c5b59..0e62c74ed48c 100644 --- a/apisix/plugins/ai-cache.lua +++ b/apisix/plugins/ai-cache.lua @@ -135,6 +135,17 @@ function _M.access(conf, ctx) return end + -- A prompt carrying non-text content (images, audio, other typed parts) + -- cannot be faithfully keyed: the protocol get_messages() flattens content + -- to plain text, so text+image collides with a text-only entry at L1 and + -- across different images at L2. Never serve or store such a request -- leave + -- the fingerprint unset so log() writes nothing back -- and report a MISS so + -- it proceeds to the upstream uncached. + if semantic.body_has_nontext(body) then + ctx.ai_cache_status = "MISS" + return + end + ctx.ai_cache_fingerprint = key_mod.fingerprint(ctx, body) ctx.ai_cache_key = key_mod.build(conf, ctx, ctx.ai_cache_fingerprint) -- Remember which instance the fingerprint was computed for. ai-proxy-multi diff --git a/apisix/plugins/ai-cache/semantic.lua b/apisix/plugins/ai-cache/semantic.lua index 4c603a9dca0e..65813bd38c0a 100644 --- a/apisix/plugins/ai-cache/semantic.lua +++ b/apisix/plugins/ai-cache/semantic.lua @@ -127,9 +127,21 @@ function _M.context_messages(messages, match) end -function _M.window_has_nontext(messages, match) - for _, i in ipairs(embed_window(messages, match)) do - local content = messages[i].content +-- True when any message in the RAW request body carries non-text content +-- (images, audio, other typed parts). This MUST read the raw body rather than +-- the protocol's canonical messages: get_messages() flattens content to plain +-- text (dropping non-text parts), so by the time a prompt reaches the canonical +-- form the image is already gone and text+image is indistinguishable from a +-- text-only prompt. Such a prompt cannot be faithfully keyed by the cache +-- (text+image would collide with text-only at L1 and across images at L2), so +-- the caller bypasses caching entirely for it. +function _M.body_has_nontext(body) + local messages = body and body.messages + if type(messages) ~= "table" then + return false + end + for _, msg in ipairs(messages) do + local content = type(msg) == "table" and msg.content if type(content) == "table" then for _, block in ipairs(content) do if type(block) == "table" and block.type and block.type ~= "text" then @@ -217,13 +229,16 @@ function _M.embed_query(conf, ctx, body) return nil end local sem = conf.semantic - local messages = key_mod.messages(ctx, body) - -- Bypass L2 when an embedded message carries non-text content (images, etc.): - -- it is absent from both the vector and the partition, so a same-text - -- different-image prompt would otherwise collide on one L2 cell. - if _M.window_has_nontext(messages, sem.match) then + -- Bypass L2 when the prompt carries non-text content (images, etc.): it is + -- absent from both the vector and the partition, so a same-text + -- different-image prompt would otherwise collide on one L2 cell. Detected + -- from the raw body because get_messages() has already flattened it away. + -- The access phase bypasses the whole cache for such a prompt; this is the + -- second line of defence should the semantic layer ever be reached directly. + if _M.body_has_nontext(body) then return nil end + local messages = key_mod.messages(ctx, body) local text = _M.extract_embed_text(messages, sem.match) if text == "" then return nil From 67392a319de8e10da1662d07526aca6caf59fea2 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Fri, 3 Jul 2026 14:15:15 +0800 Subject: [PATCH 2/2] fix(ai-cache): detect non-text content across protocol shapes body_has_nontext() only recognised OpenAI-style typed parts (type ~= "text"), so Bedrock Converse content blocks -- which mark non-text modalities with a distinct key (image/document/toolUse/...) and no `type` field -- slipped through. get_messages() then flattened them to plain text, letting a Bedrock text+image request collide with a text-only one on the same L1 exact key. Treat a block as text only when it carries a string `text` and no non-text discriminator, covering both the OpenAI `type` tag and the Bedrock key-per-modality shape. Adds a Bedrock exact-cache regression (TEST 66-68). --- apisix/plugins/ai-cache/semantic.lua | 33 +++++++++--- t/plugin/ai-cache-semantic.t | 79 ++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 8 deletions(-) diff --git a/apisix/plugins/ai-cache/semantic.lua b/apisix/plugins/ai-cache/semantic.lua index 65813bd38c0a..aed8234224bd 100644 --- a/apisix/plugins/ai-cache/semantic.lua +++ b/apisix/plugins/ai-cache/semantic.lua @@ -127,14 +127,31 @@ function _M.context_messages(messages, match) end +-- A content block is text-only iff it carries a string `text` and nothing that +-- marks it as another modality. Two block shapes exist across supported client +-- protocols, so the test must cover both: +-- * OpenAI Chat tags every part with `type` ({type="text",text=...} vs +-- {type="image_url",...}), so a `type` other than "text" is non-text. +-- * Bedrock Converse uses a distinct key per modality with no `type` +-- ({text=...} vs {image=...}/{document=...}/{toolUse=...}), so a block +-- without a string `text` is non-text there. +-- Anything that is not a plain text block (including a non-table element) is +-- treated as non-text, which is the safe default: at worst it declines to cache. +local function block_is_text(block) + return type(block) == "table" + and type(block.text) == "string" + and (block.type == nil or block.type == "text") +end + + -- True when any message in the RAW request body carries non-text content --- (images, audio, other typed parts). This MUST read the raw body rather than --- the protocol's canonical messages: get_messages() flattens content to plain --- text (dropping non-text parts), so by the time a prompt reaches the canonical --- form the image is already gone and text+image is indistinguishable from a --- text-only prompt. Such a prompt cannot be faithfully keyed by the cache --- (text+image would collide with text-only at L1 and across images at L2), so --- the caller bypasses caching entirely for it. +-- (images, documents, audio, tool blocks, ...). This MUST read the raw body +-- rather than the protocol's canonical messages: get_messages() flattens +-- content to plain text (dropping non-text parts), so by the time a prompt +-- reaches the canonical form the image is already gone and text+image is +-- indistinguishable from a text-only prompt. Such a prompt cannot be faithfully +-- keyed by the cache (text+image would collide with text-only at L1 and across +-- images at L2), so the caller bypasses caching entirely for it. function _M.body_has_nontext(body) local messages = body and body.messages if type(messages) ~= "table" then @@ -144,7 +161,7 @@ function _M.body_has_nontext(body) local content = type(msg) == "table" and msg.content if type(content) == "table" then for _, block in ipairs(content) do - if type(block) == "table" and block.type and block.type ~= "text" then + if not block_is_text(block) then return true end end diff --git a/t/plugin/ai-cache-semantic.t b/t/plugin/ai-cache-semantic.t index e7b22a7c61c1..5eff1564b2a5 100644 --- a/t/plugin/ai-cache-semantic.t +++ b/t/plugin/ai-cache-semantic.t @@ -1525,3 +1525,82 @@ X-AI-Cache-Status: MISS } --- response_body recreated-for-target-B + + + +=== TEST 66: set a bedrock-converse route (exact L1 only) for the multimodal-collision regression +--- extra_yaml_config +plugins: + - ai-proxy-multi + - ai-cache +--- config + location /t { + content_by_lua_block { + require("lib.test_redis").flush_port("127.0.0.1", 6379) + + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/ai/converse", + "plugins": { + "ai-proxy-multi": { + "instances": [ + { + "name": "bedrock", + "provider": "bedrock", + "weight": 1, + "auth": { + "aws": { + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + } + }, + "provider_conf": { "region": "us-east-1" }, + "options": { "model": "anthropic.claude-3-5-sonnet-20241022-v2:0" }, + "override": { "endpoint": "http://127.0.0.1:1980/model/anthropic.claude-3-5-sonnet-20241022-v2:0/converse" } + } + ], + "ssl_verify": false + }, + "ai-cache": { + "redis_host": "127.0.0.1", + "redis_port": 6379 + } + } + }]] + ) + if code >= 300 then ngx.status = code end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 67: a text-only bedrock prompt is a cold MISS (warms the exact L1 entry) +--- extra_yaml_config +plugins: + - ai-proxy-multi + - ai-cache +--- request +POST /ai/converse +{"messages":[{"role":"user","content":[{"text":"What is the capital of France?"}]}]} +--- response_headers +X-AI-Cache-Status: MISS +--- wait: 0.5 + + + +=== TEST 68: the SAME text alongside a bedrock image block is a MISS, not a cross-modal L1 hit +--- extra_yaml_config +plugins: + - ai-proxy-multi + - ai-cache +--- request +POST /ai/converse +{"messages":[{"role":"user","content":[{"text":"What is the capital of France?"},{"image":{"format":"png","source":{"bytes":"aW1n"}}}]}]} +--- response_headers +X-AI-Cache-Status: MISS +--- wait: 0.5