diff --git a/apisix/plugins/ai-cache.lua b/apisix/plugins/ai-cache.lua index 895ccc7c5b59..0e62c74ed48c 100644 --- a/apisix/plugins/ai-cache.lua +++ b/apisix/plugins/ai-cache.lua @@ -135,6 +135,17 @@ function _M.access(conf, ctx) return end + -- A prompt carrying non-text content (images, audio, other typed parts) + -- cannot be faithfully keyed: the protocol get_messages() flattens content + -- to plain text, so text+image collides with a text-only entry at L1 and + -- across different images at L2. Never serve or store such a request -- leave + -- the fingerprint unset so log() writes nothing back -- and report a MISS so + -- it proceeds to the upstream uncached. + if semantic.body_has_nontext(body) then + ctx.ai_cache_status = "MISS" + return + end + ctx.ai_cache_fingerprint = key_mod.fingerprint(ctx, body) ctx.ai_cache_key = key_mod.build(conf, ctx, ctx.ai_cache_fingerprint) -- Remember which instance the fingerprint was computed for. ai-proxy-multi diff --git a/apisix/plugins/ai-cache/semantic.lua b/apisix/plugins/ai-cache/semantic.lua index 4c603a9dca0e..aed8234224bd 100644 --- a/apisix/plugins/ai-cache/semantic.lua +++ b/apisix/plugins/ai-cache/semantic.lua @@ -127,12 +127,41 @@ function _M.context_messages(messages, match) end -function _M.window_has_nontext(messages, match) - for _, i in ipairs(embed_window(messages, match)) do - local content = messages[i].content +-- A content block is text-only iff it carries a string `text` and nothing that +-- marks it as another modality. Two block shapes exist across supported client +-- protocols, so the test must cover both: +-- * OpenAI Chat tags every part with `type` ({type="text",text=...} vs +-- {type="image_url",...}), so a `type` other than "text" is non-text. +-- * Bedrock Converse uses a distinct key per modality with no `type` +-- ({text=...} vs {image=...}/{document=...}/{toolUse=...}), so a block +-- without a string `text` is non-text there. +-- Anything that is not a plain text block (including a non-table element) is +-- treated as non-text, which is the safe default: at worst it declines to cache. +local function block_is_text(block) + return type(block) == "table" + and type(block.text) == "string" + and (block.type == nil or block.type == "text") +end + + +-- True when any message in the RAW request body carries non-text content +-- (images, documents, audio, tool blocks, ...). This MUST read the raw body +-- rather than the protocol's canonical messages: get_messages() flattens +-- content to plain text (dropping non-text parts), so by the time a prompt +-- reaches the canonical form the image is already gone and text+image is +-- indistinguishable from a text-only prompt. Such a prompt cannot be faithfully +-- keyed by the cache (text+image would collide with text-only at L1 and across +-- images at L2), so the caller bypasses caching entirely for it. +function _M.body_has_nontext(body) + local messages = body and body.messages + if type(messages) ~= "table" then + return false + end + for _, msg in ipairs(messages) do + local content = type(msg) == "table" and msg.content if type(content) == "table" then for _, block in ipairs(content) do - if type(block) == "table" and block.type and block.type ~= "text" then + if not block_is_text(block) then return true end end @@ -217,13 +246,16 @@ function _M.embed_query(conf, ctx, body) return nil end local sem = conf.semantic - local messages = key_mod.messages(ctx, body) - -- Bypass L2 when an embedded message carries non-text content (images, etc.): - -- it is absent from both the vector and the partition, so a same-text - -- different-image prompt would otherwise collide on one L2 cell. - if _M.window_has_nontext(messages, sem.match) then + -- Bypass L2 when the prompt carries non-text content (images, etc.): it is + -- absent from both the vector and the partition, so a same-text + -- different-image prompt would otherwise collide on one L2 cell. Detected + -- from the raw body because get_messages() has already flattened it away. + -- The access phase bypasses the whole cache for such a prompt; this is the + -- second line of defence should the semantic layer ever be reached directly. + if _M.body_has_nontext(body) then return nil end + local messages = key_mod.messages(ctx, body) local text = _M.extract_embed_text(messages, sem.match) if text == "" then return nil diff --git a/t/plugin/ai-cache-semantic.t b/t/plugin/ai-cache-semantic.t index e7b22a7c61c1..5eff1564b2a5 100644 --- a/t/plugin/ai-cache-semantic.t +++ b/t/plugin/ai-cache-semantic.t @@ -1525,3 +1525,82 @@ X-AI-Cache-Status: MISS } --- response_body recreated-for-target-B + + + +=== TEST 66: set a bedrock-converse route (exact L1 only) for the multimodal-collision regression +--- extra_yaml_config +plugins: + - ai-proxy-multi + - ai-cache +--- config + location /t { + content_by_lua_block { + require("lib.test_redis").flush_port("127.0.0.1", 6379) + + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/ai/converse", + "plugins": { + "ai-proxy-multi": { + "instances": [ + { + "name": "bedrock", + "provider": "bedrock", + "weight": 1, + "auth": { + "aws": { + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + } + }, + "provider_conf": { "region": "us-east-1" }, + "options": { "model": "anthropic.claude-3-5-sonnet-20241022-v2:0" }, + "override": { "endpoint": "http://127.0.0.1:1980/model/anthropic.claude-3-5-sonnet-20241022-v2:0/converse" } + } + ], + "ssl_verify": false + }, + "ai-cache": { + "redis_host": "127.0.0.1", + "redis_port": 6379 + } + } + }]] + ) + if code >= 300 then ngx.status = code end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 67: a text-only bedrock prompt is a cold MISS (warms the exact L1 entry) +--- extra_yaml_config +plugins: + - ai-proxy-multi + - ai-cache +--- request +POST /ai/converse +{"messages":[{"role":"user","content":[{"text":"What is the capital of France?"}]}]} +--- response_headers +X-AI-Cache-Status: MISS +--- wait: 0.5 + + + +=== TEST 68: the SAME text alongside a bedrock image block is a MISS, not a cross-modal L1 hit +--- extra_yaml_config +plugins: + - ai-proxy-multi + - ai-cache +--- request +POST /ai/converse +{"messages":[{"role":"user","content":[{"text":"What is the capital of France?"},{"image":{"format":"png","source":{"bytes":"aW1n"}}}]}]} +--- response_headers +X-AI-Cache-Status: MISS +--- wait: 0.5