diff --git a/astrbot/core/agent/message.py b/astrbot/core/agent/message.py index 4292f4c04e..6a7e523681 100644 --- a/astrbot/core/agent/message.py +++ b/astrbot/core/agent/message.py @@ -302,6 +302,71 @@ def get_checkpoint_id(message: Message | dict) -> str | None: return None +_IMAGE_HISTORY_PLACEHOLDER = "[Image]" + + +def _is_image_content_part(part: Any) -> bool: + if isinstance(part, ImageURLPart): + return True + if isinstance(part, dict) and part.get("type") == "image_url": + return True + return getattr(part, "type", None) == "image_url" + + +def _image_placeholder_part_dict() -> dict[str, Any]: + return {"type": "text", "text": _IMAGE_HISTORY_PLACEHOLDER} + + +def strip_images_from_content_parts(content: Any) -> Any: + """Replace image parts with a short text placeholder. + + Used when persisting or reloading conversation history so previous-turn + image pixels / remote URLs do not keep leaking into later requests. + """ + if not isinstance(content, list): + return content + + stripped: list[Any] = [] + previous_was_image_placeholder = False + for part in content: + if _is_image_content_part(part): + if previous_was_image_placeholder: + continue + if isinstance(part, ContentPart): + stripped.append(TextPart(text=_IMAGE_HISTORY_PLACEHOLDER)) + else: + stripped.append(_image_placeholder_part_dict()) + previous_was_image_placeholder = True + continue + + previous_was_image_placeholder = False + stripped.append(part) + return stripped + + +def strip_images_from_history_message_dict(message: dict[str, Any]) -> dict[str, Any]: + """Return a shallow-copied history message dict without image payloads.""" + if not isinstance(message, dict): + return message + if is_checkpoint_message(message): + return message + + content = message.get("content") + if not isinstance(content, list): + return message + + new_message = dict(message) + new_message["content"] = strip_images_from_content_parts(content) + return new_message + + +def strip_images_from_history_messages(history: list[dict] | None) -> list[dict]: + """Strip image payloads from a full persisted history list.""" + if not history: + return [] + return [strip_images_from_history_message_dict(item) for item in history] + + def strip_checkpoint_messages(history: list[dict]) -> list[dict]: """Remove internal checkpoint messages from provider-facing history.""" return [message for message in history if not is_checkpoint_message(message)] @@ -327,7 +392,8 @@ def _get_checkpoint_data(message: Message | dict) -> CheckpointData | None: def bind_checkpoint_messages(history: list[dict]) -> list[Message]: """Load persisted history and bind checkpoint segments to prior messages.""" messages: list[Message] = [] - for item in history: + for raw_item in history: + item = strip_images_from_history_message_dict(raw_item) if is_checkpoint_message(item): checkpoint = _get_checkpoint_data(item) if checkpoint is not None and messages: @@ -348,10 +414,14 @@ def dump_messages_with_checkpoints(messages: list[Message]) -> list[dict]: for message in messages: message_data = message.model_dump() if isinstance(message.content, list): + # Drop ephemeral parts first, then reuse shared image-stripping rules. + filtered_parts = [ + part for part in message.content if not getattr(part, "_no_save", False) + ] + stripped_parts = strip_images_from_content_parts(filtered_parts) message_data["content"] = [ - part.model_dump() - for part in message.content - if not getattr(part, "_no_save", False) + part.model_dump() if isinstance(part, ContentPart) else part + for part in stripped_parts ] dumped.append(message_data) if message._checkpoint_after is not None: diff --git a/astrbot/core/astr_main_agent.py b/astrbot/core/astr_main_agent.py index 1eae0f17e3..826a35c682 100644 --- a/astrbot/core/astr_main_agent.py +++ b/astrbot/core/astr_main_agent.py @@ -14,7 +14,7 @@ from astrbot.core import logger from astrbot.core.agent.handoff import HandoffTool from astrbot.core.agent.mcp_client import MCPTool -from astrbot.core.agent.message import TextPart +from astrbot.core.agent.message import TextPart, strip_images_from_history_messages from astrbot.core.agent.tool import ToolSet from astrbot.core.astr_agent_context import AgentContextWrapper, AstrAgentContext from astrbot.core.astr_agent_hooks import MAIN_AGENT_HOOKS @@ -1351,6 +1351,44 @@ def _select_image_chat_provider( return provider +def _is_remote_image_ref(image_ref: str) -> bool: + lowered = image_ref.lower() + return lowered.startswith("http://") or lowered.startswith("https://") + + +def _finalize_request_image_urls( + image_urls: list[str] | None, + *, + max_total: int = 4, +) -> list[str]: + """Dedupe and prioritize local image refs over remote URLs. + + Remote CDN links (especially QQ multimedia URLs) expire easily. Keeping them + beside already-downloaded local paths causes models to "see" missing/wrong + images in later turns. + """ + urls = normalize_and_dedupe_strings(image_urls or []) + if not urls: + return [] + + local_refs: list[str] = [] + embedded_refs: list[str] = [] + remote_refs: list[str] = [] + for ref in urls: + lowered = ref.lower() + if _is_remote_image_ref(ref): + remote_refs.append(ref) + elif lowered.startswith("data:") or lowered.startswith("base64://"): + embedded_refs.append(ref) + else: + local_refs.append(ref) + + ordered = local_refs + embedded_refs + remote_refs + if max_total > 0: + ordered = ordered[:max_total] + return ordered + + async def build_main_agent( *, event: AstrMessageEvent, @@ -1381,7 +1419,9 @@ async def build_main_agent( "provider_request 必须是 ProviderRequest 类型。" ) if req.conversation: - req.contexts = json.loads(req.conversation.history) + req.contexts = strip_images_from_history_messages( + json.loads(req.conversation.history) + ) else: req = ProviderRequest() req.prompt = "" @@ -1514,7 +1554,9 @@ async def build_main_agent( conversation = await _get_session_conv(event, plugin_context) req.conversation = conversation - req.contexts = json.loads(conversation.history) + req.contexts = strip_images_from_history_messages( + json.loads(conversation.history) + ) event.set_extra("provider_request", req) if isinstance(req.contexts, str): @@ -1530,7 +1572,12 @@ async def build_main_agent( ) ) ) - req.image_urls = normalize_and_dedupe_strings(req.image_urls) + raw_max_quoted_fallback_images = getattr(config, "max_quoted_fallback_images", 4) + max_quoted_fallback_images = max(int(raw_max_quoted_fallback_images or 4), 4) + req.image_urls = _finalize_request_image_urls( + req.image_urls, + max_total=max_quoted_fallback_images, + ) req.audio_urls = normalize_and_dedupe_strings(req.audio_urls) if config.file_extract_enabled: diff --git a/astrbot/core/provider/sources/openai_source.py b/astrbot/core/provider/sources/openai_source.py index b1594d608a..88456bc39d 100644 --- a/astrbot/core/provider/sources/openai_source.py +++ b/astrbot/core/provider/sources/openai_source.py @@ -43,6 +43,8 @@ from ..register import register_provider_adapter from .request_retry import retry_provider_request +IMAGE_UNAVAILABLE_PLACEHOLDER = "[Image unavailable]" + @register_provider_adapter( "openai_chat_completion", @@ -273,7 +275,10 @@ async def _transform_content_part(self, part: dict) -> dict: if part.get("type") == "image_url": url, image_detail = self._extract_image_part_info(part) if not url: - return part + return { + "type": "text", + "text": IMAGE_UNAVAILABLE_PLACEHOLDER, + } try: resolved_part = await self._resolve_image_part( @@ -281,13 +286,26 @@ async def _transform_content_part(self, part: dict) -> dict: ) except Exception as exc: logger.warning( - "图片 %s 预处理失败，将保留原始内容。错误: %s", + "Image preprocess failed; dropping image content. url=%s err=%s", url, exc, ) - return part + return { + "type": "text", + "text": IMAGE_UNAVAILABLE_PLACEHOLDER, + } - return resolved_part or part + if resolved_part: + return resolved_part + + logger.warning( + "Image preprocess returned empty; dropping image content. url=%s", + url, + ) + return { + "type": "text", + "text": IMAGE_UNAVAILABLE_PLACEHOLDER, + } if part.get("type") == "audio_url": audio_ref = self._extract_audio_part_info(part) diff --git a/tests/test_image_history_strip.py b/tests/test_image_history_strip.py new file mode 100644 index 0000000000..3b2f63d5e6 --- /dev/null +++ b/tests/test_image_history_strip.py @@ -0,0 +1,67 @@ +from astrbot.core.agent.message import ( + ImageURLPart, + Message, + TextPart, + bind_checkpoint_messages, + dump_messages_with_checkpoints, + strip_images_from_history_messages, +) +from astrbot.core.astr_main_agent import _finalize_request_image_urls + + +def test_dump_messages_strips_image_parts(): + msg = Message( + role="user", + content=[ + TextPart(text="see this"), + ImageURLPart( + image_url=ImageURLPart.ImageURL(url="data:image/png;base64,AAAA") + ), + ImageURLPart( + image_url=ImageURLPart.ImageURL(url="https://example.com/x.jpg") + ), + ], + ) + dumped = dump_messages_with_checkpoints([msg]) + content = dumped[0]["content"] + assert all(part.get("type") != "image_url" for part in content) + assert sum(1 for part in content if part.get("text") == "[Image]") == 1 + + +def test_bind_checkpoint_messages_strips_polluted_history(): + hist = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "old"}, + { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,BBB"}, + }, + { + "type": "image_url", + "image_url": {"url": "https://example.com/a.jpg"}, + }, + ], + } + ] + bound = bind_checkpoint_messages(hist) + types = [getattr(part, "type", None) for part in bound[0].content] + assert "image_url" not in types + stripped = strip_images_from_history_messages(hist) + assert all(part.get("type") != "image_url" for part in stripped[0]["content"]) + + +def test_finalize_request_image_urls_prefers_local_and_limits(): + out = _finalize_request_image_urls( + [ + "https://example.com/a", + "C:/tmp/a.jpg", + "HTTPS://example.com/a", + "C:/tmp/b.jpg", + "https://example.com/c.jpg", + "data:image/png;base64,AAAA", + ], + max_total=2, + ) + assert out == ["C:/tmp/a.jpg", "C:/tmp/b.jpg"] diff --git a/tests/test_openai_source.py b/tests/test_openai_source.py index a45a232938..8cfbcff1fa 100644 --- a/tests/test_openai_source.py +++ b/tests/test_openai_source.py @@ -1217,7 +1217,7 @@ def fake_warning(message, *args, **kwargs): @pytest.mark.asyncio -async def test_prepare_chat_payload_keeps_original_context_image_when_materialization_fails( +async def test_prepare_chat_payload_drops_context_image_when_materialization_fails( monkeypatch, ): provider = _make_provider() @@ -1261,10 +1261,8 @@ async def fake_resolve_media_ref_to_base64_data( assert payloads["messages"][0]["content"] == [ {"type": "text", "text": "look"}, { - "type": "image_url", - "image_url": { - "url": "https://example.com/expired.png", - }, + "type": "text", + "text": "[Image unavailable]", }, ] finally: