fix(ai-cache): bypass caching for prompts carrying non-text content

apisix/plugins/ai-cache.lua

@@ -135,6 +135,17 @@ function _M.access(conf, ctx)
         return
     end
 
+    -- A prompt carrying non-text content (images, audio, other typed parts)
+    -- cannot be faithfully keyed: the protocol get_messages() flattens content
+    -- to plain text, so text+image collides with a text-only entry at L1 and
+    -- across different images at L2. Never serve or store such a request -- leave
+    -- the fingerprint unset so log() writes nothing back -- and report a MISS so
+    -- it proceeds to the upstream uncached.
+    if semantic.body_has_nontext(body) then
+        ctx.ai_cache_status = "MISS"
+        return
+    end
+
     ctx.ai_cache_fingerprint = key_mod.fingerprint(ctx, body)
     ctx.ai_cache_key = key_mod.build(conf, ctx, ctx.ai_cache_fingerprint)
     -- Remember which instance the fingerprint was computed for. ai-proxy-multi

apisix/plugins/ai-cache/semantic.lua

@@ -127,12 +127,41 @@ function _M.context_messages(messages, match)
 end
 
 
-function _M.window_has_nontext(messages, match)
-    for _, i in ipairs(embed_window(messages, match)) do
-        local content = messages[i].content
+-- A content block is text-only iff it carries a string `text` and nothing that
+-- marks it as another modality. Two block shapes exist across supported client
+-- protocols, so the test must cover both:
+--   * OpenAI Chat tags every part with `type` ({type="text",text=...} vs
+--     {type="image_url",...}), so a `type` other than "text" is non-text.
+--   * Bedrock Converse uses a distinct key per modality with no `type`
+--     ({text=...} vs {image=...}/{document=...}/{toolUse=...}), so a block
+--     without a string `text` is non-text there.
+-- Anything that is not a plain text block (including a non-table element) is
+-- treated as non-text, which is the safe default: at worst it declines to cache.
+local function block_is_text(block)
+    return type(block) == "table"
+       and type(block.text) == "string"
+       and (block.type == nil or block.type == "text")
+end
+
+
+-- True when any message in the RAW request body carries non-text content
+-- (images, documents, audio, tool blocks, ...). This MUST read the raw body
+-- rather than the protocol's canonical messages: get_messages() flattens
+-- content to plain text (dropping non-text parts), so by the time a prompt
+-- reaches the canonical form the image is already gone and text+image is
+-- indistinguishable from a text-only prompt. Such a prompt cannot be faithfully
+-- keyed by the cache (text+image would collide with text-only at L1 and across
+-- images at L2), so the caller bypasses caching entirely for it.
+function _M.body_has_nontext(body)
+    local messages = body and body.messages
+    if type(messages) ~= "table" then
+        return false
+    end
+    for _, msg in ipairs(messages) do
+        local content = type(msg) == "table" and msg.content
         if type(content) == "table" then
             for _, block in ipairs(content) do
-                if type(block) == "table" and block.type and block.type ~= "text" then
+                if not block_is_text(block) then
                     return true
                 end
             end
@@ -217,13 +246,16 @@ function _M.embed_query(conf, ctx, body)
         return nil
     end
     local sem      = conf.semantic
-    local messages = key_mod.messages(ctx, body)
-    -- Bypass L2 when an embedded message carries non-text content (images, etc.):
-    -- it is absent from both the vector and the partition, so a same-text
-    -- different-image prompt would otherwise collide on one L2 cell.
-    if _M.window_has_nontext(messages, sem.match) then
+    -- Bypass L2 when the prompt carries non-text content (images, etc.): it is
+    -- absent from both the vector and the partition, so a same-text
+    -- different-image prompt would otherwise collide on one L2 cell. Detected
+    -- from the raw body because get_messages() has already flattened it away.
+    -- The access phase bypasses the whole cache for such a prompt; this is the
+    -- second line of defence should the semantic layer ever be reached directly.
+    if _M.body_has_nontext(body) then
         return nil
     end
+    local messages = key_mod.messages(ctx, body)
     local text     = _M.extract_embed_text(messages, sem.match)
     if text == "" then
         return nil

t/plugin/ai-cache-semantic.t

@@ -1525,3 +1525,82 @@ X-AI-Cache-Status: MISS
     }
 --- response_body
 recreated-for-target-B
+
+
+
+=== TEST 66: set a bedrock-converse route (exact L1 only) for the multimodal-collision regression
+--- extra_yaml_config
+plugins:
+  - ai-proxy-multi
+  - ai-cache
+--- config
+    location /t {
+        content_by_lua_block {
+            require("lib.test_redis").flush_port("127.0.0.1", 6379)
+
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                ngx.HTTP_PUT,
+                [[{
+                    "uri": "/ai/converse",
+                    "plugins": {
+                        "ai-proxy-multi": {
+                            "instances": [
+                                {
+                                    "name": "bedrock",
+                                    "provider": "bedrock",
+                                    "weight": 1,
+                                    "auth": {
+                                        "aws": {
+                                            "access_key_id": "AKIAIOSFODNN7EXAMPLE",
+                                            "secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+                                        }
+                                    },
+                                    "provider_conf": { "region": "us-east-1" },
+                                    "options": { "model": "anthropic.claude-3-5-sonnet-20241022-v2:0" },
+                                    "override": { "endpoint": "http://127.0.0.1:1980/model/anthropic.claude-3-5-sonnet-20241022-v2:0/converse" }
+                                }
+                            ],
+                            "ssl_verify": false
+                        },
+                        "ai-cache": {
+                            "redis_host": "127.0.0.1",
+                            "redis_port": 6379
+                        }
+                    }
+                }]]
+            )
+            if code >= 300 then ngx.status = code end
+            ngx.say(body)
+        }
+    }
+--- response_body
+passed
+
+
+
+=== TEST 67: a text-only bedrock prompt is a cold MISS (warms the exact L1 entry)
+--- extra_yaml_config
+plugins:
+  - ai-proxy-multi
+  - ai-cache
+--- request
+POST /ai/converse
+{"messages":[{"role":"user","content":[{"text":"What is the capital of France?"}]}]}
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 0.5
+
+
+
+=== TEST 68: the SAME text alongside a bedrock image block is a MISS, not a cross-modal L1 hit
+--- extra_yaml_config
+plugins:
+  - ai-proxy-multi
+  - ai-cache
+--- request
+POST /ai/converse
+{"messages":[{"role":"user","content":[{"text":"What is the capital of France?"},{"image":{"format":"png","source":{"bytes":"aW1n"}}}]}]}
+--- response_headers
+X-AI-Cache-Status: MISS
+--- wait: 0.5