From 3ea70a805505b07fd7220fade1e4200c333080a3 Mon Sep 17 00:00:00 2001 From: Claire-ovo <132752951+Claire-ovo@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:49:03 +0800 Subject: [PATCH 1/2] feat: serialize index configurations with safe connection metadata --- docs/api/searchindex.rst | 58 +++++ redisvl/index/index.py | 160 +++++++++++++- tests/unit/test_index_serialization.py | 281 +++++++++++++++++++++++++ 3 files changed, 493 insertions(+), 6 deletions(-) create mode 100644 tests/unit/test_index_serialization.py diff --git a/docs/api/searchindex.rst b/docs/api/searchindex.rst index 56a24ae4..8ee5e64d 100644 --- a/docs/api/searchindex.rst +++ b/docs/api/searchindex.rst @@ -17,6 +17,64 @@ Search Index Classes * - :ref:`bulkresult_api` - Outcome of a bulk operation: how many documents matched, how many were processed, and whether the run completed. +Serializing index configurations +=============================== + +Both index classes provide synchronous ``to_dict()`` and ``to_yaml()`` methods. +They export the schema by default, without opening a connection or reading +``REDIS_URL``. Existing schema-only ``from_dict()`` and ``from_yaml()`` calls +continue to work. + +.. code-block:: python + + config = index.to_dict() + restored = SearchIndex.from_dict(config, redis_url="redis://localhost:6379") + index.to_yaml("index.yaml", overwrite=False) + +Pass ``include_connection=True`` to include a ``connection`` mapping alongside +``index``, ``fields`` and ``version``: + +.. code-block:: yaml + + connection: + redis_url: redis://localhost:6379/0 + connection_kwargs: + socket_timeout: 2.0 + requires_override: false + +Connection exports retain the URL endpoint and path, including IPv6 brackets +and Sentinel host lists. They omit URL user information, **all** query +parameters and fragments. Only boolean ``ssl``, ``ssl_check_hostname``, +``decode_responses``, ``socket_keepalive``, ``retry_on_timeout`` and ``cluster``; +finite numeric ``db``, ``protocol``, ``socket_timeout``, +``socket_connect_timeout``, ``health_check_interval`` and ``max_connections``; +and ``ssl_cert_reqs`` values ``none``, ``optional`` or ``required`` are exported +from connection kwargs. Other options, such as passwords, certificate paths, +credential providers and custom connection pools, are omitted. + +Any omission sets ``requires_override: true``. Configurations sourced from an +injected client or the environment also require an override: their connection +internals are not inspected. Restoring such a configuration without a nonempty +``redis_url`` or a ``redis_client`` raises ``ValueError``, even if ``REDIS_URL`` +is set. Supply the complete connection configuration at restore time: + +.. code-block:: python + + import os + + config = index.to_dict(include_connection=True) + restored = SearchIndex.from_dict(config, redis_url=os.environ["REDIS_URL"]) + +A replacement URL or client discards stored connection kwargs; pass any required +``connection_kwargs`` explicitly. Without a replacement endpoint, explicit +``connection_kwargs`` replace the stored mapping rather than merging with it. +Unredacted configurations restore their stored endpoint and allowed options. +YAML imports use the same rules. + +This is configuration serialization, not a snapshot of live Redis data or +client state. Endpoints, paths and the schema may themselves be private: use the +default schema-only export when sharing artifacts outside your deployment. + .. _searchindex_api: SearchIndex diff --git a/redisvl/index/index.py b/redisvl/index/index.py index 6a2451fc..706b1381 100644 --- a/redisvl/index/index.py +++ b/redisvl/index/index.py @@ -6,7 +6,8 @@ import warnings import weakref from dataclasses import dataclass -from math import ceil +from math import ceil, isfinite +from pathlib import Path from typing import ( TYPE_CHECKING, Any, @@ -19,8 +20,10 @@ Sequence, cast, ) +from urllib.parse import urlsplit import redis.exceptions +import yaml # Add missing imports from redis import Redis @@ -671,6 +674,9 @@ class BaseSearchIndex: } schema: IndexSchema + _redis_url: str | None + _connection_kwargs: dict[str, Any] + _client_was_injected: bool # Module-level callback (set per subclass) used to close an owned client # when the index is garbage collected. Must never be a bound method. @@ -859,9 +865,114 @@ def storage_type(self) -> StorageType: hash or json.""" return self.schema.index.storage_type + def to_dict(self, include_connection: bool = False) -> dict[str, Any]: + """Export the schema and, optionally, safe connection metadata. + + No Redis connection is opened. By default the result is identical to + ``schema.to_dict()``. Connection exports omit URL user information, + query parameters, fragments and non-allowlisted connection options. + Any omission requires a complete explicit connection when restoring. + Injected clients and environment variables are never inspected. + + Args: + include_connection: Include a ``connection`` mapping. Defaults to False. + + Returns: + A dictionary accepted by ``from_dict()``. + + Raises: + ValueError: If an explicitly configured URL cannot be safely exported. + """ + data = self.schema.to_dict() + if not include_connection: + return data + + url = None if self._client_was_injected else self._redis_url + requires_override = not url + if url: + try: + parts = urlsplit(url) + if parts.scheme not in {"redis", "rediss", "unix", "redis+sentinel"}: + raise ValueError + endpoint = parts.netloc.rsplit("@", 1)[-1] + if parts.scheme != "unix" and not endpoint: + raise ValueError + except ValueError: + raise ValueError( + "Cannot safely serialize the configured Redis URL" + ) from None + requires_override = bool( + "@" in parts.netloc or parts.query or parts.fragment + ) + # Keep the authority intact: rebuilding from hostname loses IPv6 + # brackets and Sentinel host lists. + url = f"{parts.scheme}://{endpoint}{parts.path}" + + safe_kwargs: dict[str, Any] = {} + boolean_options = { + "ssl", + "ssl_check_hostname", + "decode_responses", + "socket_keepalive", + "retry_on_timeout", + "cluster", + } + numeric_options = { + "db", + "protocol", + "socket_timeout", + "socket_connect_timeout", + "health_check_interval", + "max_connections", + } + for key, value in self._connection_kwargs.items(): + if ( + (key in boolean_options and type(value) is bool) + or ( + key in numeric_options + and type(value) in (int, float) + and isfinite(value) + ) + or ( + key == "ssl_cert_reqs" + and type(value) is str + and value in {"none", "optional", "required"} + ) + ): + safe_kwargs[key] = value + else: + requires_override = True + data["connection"] = { + "redis_url": url, + "connection_kwargs": safe_kwargs, + "requires_override": requires_override, + } + return data + + def to_yaml( + self, file_path: str, include_connection: bool = False, overwrite: bool = True + ) -> None: + """Write the configuration to YAML, with the same rules as ``to_dict``. + + Args: + file_path: Destination file. + include_connection: Include safe connection metadata. Defaults to False. + overwrite: Replace an existing file. Defaults to True. + + Raises: + FileExistsError: If the destination exists and overwrite is False. + ValueError: If a configured URL cannot be safely exported. + """ + # Serialize before opening the file so validation cannot truncate it. + content = yaml.safe_dump(self.to_dict(include_connection), sort_keys=False) + with open(Path(file_path).resolve(), "w" if overwrite else "x") as file: + file.write(content) + @classmethod def from_yaml(cls, schema_path: str, **kwargs): - """Create a SearchIndex from a YAML schema file. + """Create a SearchIndex from a YAML schema or exported configuration. + + Connection metadata follows the same override rules as ``from_dict``. Args: schema_path (str): Path to the YAML schema file. @@ -875,12 +986,27 @@ def from_yaml(cls, schema_path: str, **kwargs): index = SearchIndex.from_yaml("schemas/schema.yaml", redis_url="redis://localhost:6379") """ - schema = IndexSchema.from_yaml(schema_path) - return cls(schema=schema, **kwargs) + try: + path = Path(schema_path).resolve() + except OSError as error: + raise ValueError(f"Invalid file path: {schema_path}") from error + if not path.exists(): + raise FileNotFoundError(f"Schema file {schema_path} does not exist") + with open(path, "r") as file: + data = yaml.safe_load(file) + if not isinstance(data, dict): + return cls(schema=IndexSchema.model_validate(data), **kwargs) + return cls.from_dict(data, **kwargs) @classmethod def from_dict(cls, schema_dict: dict[str, Any], **kwargs): - """Create a SearchIndex from a dictionary. + """Create a SearchIndex from a schema or exported configuration. + + A nonempty ``redis_url`` or an explicit ``redis_client`` replaces all + stored connection metadata. Otherwise stored options are used, with + explicit ``connection_kwargs`` replacing the stored mapping. Redacted + or unavailable connections require an explicit URL or client; they + never silently fall back to ``REDIS_URL``. Args: schema_dict (Dict[str, Any]): A dictionary containing the schema. @@ -904,7 +1030,29 @@ def from_dict(cls, schema_dict: dict[str, Any], **kwargs): }, redis_url="redis://localhost:6379") """ - schema = IndexSchema.from_dict(schema_dict) + data = schema_dict.copy() + if "connection" in data: + connection = data.pop("connection") + if not isinstance(connection, dict): + raise ValueError("connection must be a mapping") + url = connection.get("redis_url") + options = connection.get("connection_kwargs", {}) + requires_override = connection.get("requires_override", False) + if ( + (url is not None and not isinstance(url, str)) + or not isinstance(options, dict) + or type(requires_override) is not bool + ): + raise ValueError("Invalid connection metadata") + if kwargs.get("redis_client") is None and not kwargs.get("redis_url"): + if requires_override or not url: + raise ValueError( + "This configuration requires an explicit redis_url or redis_client; " + "supply the complete connection configuration." + ) + kwargs["redis_url"] = url + kwargs.setdefault("connection_kwargs", options.copy()) + schema = IndexSchema.from_dict(data) return cls(schema=schema, **kwargs) def disconnect(self): diff --git a/tests/unit/test_index_serialization.py b/tests/unit/test_index_serialization.py new file mode 100644 index 00000000..b6c2a4df --- /dev/null +++ b/tests/unit/test_index_serialization.py @@ -0,0 +1,281 @@ +"""Portable index configurations must not carry credentials or open clients.""" + +import json +from copy import deepcopy +from unittest.mock import Mock + +import pytest +import yaml + +from redisvl.index import AsyncSearchIndex, SearchIndex +from redisvl.redis.connection import RedisConnectionFactory +from redisvl.schema import IndexSchema + + +@pytest.fixture(params=[SearchIndex, AsyncSearchIndex]) +def index_class(request): + return request.param + + +@pytest.fixture +def schema(): + return { + "index": {"name": "serialized", "prefix": "documents"}, + "fields": [ + {"name": "text", "type": "text"}, + { + "name": "embedding", + "type": "vector", + "attrs": {"algorithm": "flat", "dims": 3, "distance_metric": "cosine"}, + }, + ], + } + + +@pytest.fixture(autouse=True) +def no_connections(monkeypatch): + def fail(*args, **kwargs): + pytest.fail("Serialization must not create or validate a Redis connection") + + for name in ( + "get_redis_connection", + "get_async_redis_connection", + "_get_aredis_connection", + "validate_sync_redis", + "validate_async_redis", + ): + monkeypatch.setattr(RedisConnectionFactory, name, fail) + + +def test_default_export_is_schema_only(index_class, schema, tmp_path): + index = index_class.from_dict(schema, redis_url="redis://user:secret@host:6379") + config = index.to_dict() + assert config == index.schema.to_dict() + assert index_class.from_dict(config).schema == index.schema + path = tmp_path / "schema.yaml" + index.to_yaml(path) + assert yaml.safe_load(path.read_text()) == config + assert index_class.from_yaml(path).schema == index.schema + assert "secret" not in path.read_text() + + +def test_json_schema_roundtrip(index_class, schema): + schema["index"]["storage_type"] = "json" + schema["fields"][0]["path"] = "$.text" + schema["fields"][1]["path"] = "$.embedding" + index = index_class.from_dict(schema) + assert index_class.from_dict(index.to_dict()).schema == index.schema + + +@pytest.mark.parametrize( + "url", + [ + "redis://localhost:6379/2", + "rediss://[2001:db8::1]:6380/3", + "unix:///tmp/redis.sock", + "redis+sentinel://host1:26379,host2:26380/my%2Dmaster/1", + ], +) +def test_connection_roundtrip(index_class, schema, tmp_path, url): + options = { + "socket_timeout": 2.5, + "decode_responses": True, + "ssl_cert_reqs": "required", + } + index = index_class.from_dict(schema, redis_url=url, connection_kwargs=options) + config = index.to_dict(include_connection=True) + original = deepcopy(config) + restored = index_class.from_dict(config) + assert restored._redis_url == url + assert restored._connection_kwargs == options + assert restored.schema == index.schema + assert config == original + path = tmp_path / "index.yaml" + index.to_yaml(path, include_connection=True) + assert index_class.from_yaml(path).to_dict(include_connection=True) == config + restored._connection_kwargs["socket_timeout"] = 10 + assert config == original + assert options["socket_timeout"] == 2.5 + + +@pytest.mark.parametrize( + "url,endpoint", + [ + ("redis://alice:p%40ss@[::1]:6379/0", "redis://[::1]:6379/0"), + ("redis://alice:@host/0", "redis://host/0"), + ("redis://alice@host/0", "redis://host/0"), + ("redis://host/0?password=p%40ss&db=2", "redis://host/0"), + ("rediss://host/0?ssl_password=p%40ss", "rediss://host/0"), + ("redis://host/0#p%40ss", "redis://host/0"), + ( + "redis+sentinel://alice:p%40ss@a:26379,b:26379/master/1", + "redis+sentinel://a:26379,b:26379/master/1", + ), + ("unix://alice:p%40ss@/tmp/redis.sock?db=2", "unix:///tmp/redis.sock"), + ], +) +def test_redaction_requires_explicit_restore( + index_class, schema, tmp_path, monkeypatch, url, endpoint +): + options = { + "password": "kw-secret", + "credential_provider": Mock(), + "socket_timeout": 1, + } + index = index_class.from_dict(schema, redis_url=url, connection_kwargs=options) + config = index.to_dict(include_connection=True) + assert config["connection"] == { + "redis_url": endpoint, + "connection_kwargs": {"socket_timeout": 1}, + "requires_override": True, + } + path = tmp_path / "safe.yaml" + index.to_yaml(path, include_connection=True) + for secret in ("alice", "p%40ss", "kw-secret", "credential_provider"): + assert secret not in json.dumps(config) + assert secret not in path.read_text() + monkeypatch.setenv("REDIS_URL", "redis://wrong-host:6379") + with pytest.raises(ValueError, match="explicit redis_url or redis_client"): + index_class.from_dict(config) + with pytest.raises(ValueError, match="explicit redis_url or redis_client"): + index_class.from_yaml(path) + restored = index_class.from_dict(config, redis_url=url, connection_kwargs=options) + assert restored._redis_url == url + assert restored._connection_kwargs == options + + +@pytest.mark.parametrize( + "override", + [ + {"redis_url": None}, + {"redis_url": ""}, + {"connection_kwargs": {"password": "new"}}, + ], +) +def test_incomplete_override_does_not_fall_back(index_class, schema, override): + index = index_class.from_dict(schema, redis_url="redis://user:secret@host") + with pytest.raises(ValueError, match="explicit redis_url or redis_client"): + index_class.from_dict(index.to_dict(include_connection=True), **override) + + +def test_explicit_endpoint_does_not_inherit_connection_options(index_class, schema): + index = index_class.from_dict( + schema, redis_url="rediss://original", connection_kwargs={"ssl": True} + ) + config = index.to_dict(include_connection=True) + restored = index_class.from_dict(config, redis_url="redis://other") + assert restored._redis_url == "redis://other" + assert restored._connection_kwargs == {} + client = Mock() + restored = index_class.from_dict(config, redis_client=client) + assert restored._client_was_injected + assert restored._redis_url is None + assert restored._connection_kwargs == {} + + +def test_explicit_connection_kwargs_replace_stored_options(index_class, schema): + index = index_class.from_dict( + schema, redis_url="redis://host", connection_kwargs={"socket_timeout": 1} + ) + restored = index_class.from_dict( + index.to_dict(include_connection=True), connection_kwargs={"protocol": 3} + ) + assert restored._redis_url == "redis://host" + assert restored._connection_kwargs == {"protocol": 3} + + +@pytest.mark.parametrize("injected", [False, True]) +def test_unavailable_connection_metadata_requires_replacement( + index_class, schema, monkeypatch, injected +): + monkeypatch.setenv("REDIS_URL", "redis://env-user:env-secret@host") + client = Mock() if injected else None + index = index_class.from_dict( + schema, redis_client=client, redis_url="redis://unused" if injected else None + ) + config = index.to_dict(include_connection=True) + assert config["connection"]["redis_url"] is None + assert config["connection"]["requires_override"] is True + assert "env-secret" not in json.dumps(config) + if client is not None: + assert client.mock_calls == [] + with pytest.raises(ValueError, match="explicit redis_url or redis_client"): + index_class.from_dict(config) + assert index_class.from_dict(config, redis_client=Mock())._client_was_injected + + +@pytest.mark.parametrize( + "options", + [ + {"ssl_cert_reqs": "secret"}, + {"socket_timeout": "secret"}, + {"decode_responses": Mock()}, + {"sentinel_kwargs": {"password": "secret"}}, + ], +) +def test_allowlist_rejects_unsafe_values(index_class, schema, options): + index = index_class.from_dict( + schema, redis_url="redis://host", connection_kwargs=options + ) + config = index.to_dict(include_connection=True) + assert config["connection"]["connection_kwargs"] == {} + assert config["connection"]["requires_override"] is True + assert "secret" not in json.dumps(config) + + +@pytest.mark.parametrize( + "url", + ["rediss://[invalid-secret", "http://user:secret@host", "redis:///missing-host"], +) +def test_invalid_url_errors_do_not_expose_input(index_class, schema, url): + index = index_class.from_dict(schema, redis_url=url) + with pytest.raises(ValueError) as error: + index.to_dict(include_connection=True) + assert url not in str(error.value) + assert "secret" not in str(error.value) + + +def test_export_error_preserves_existing_file(index_class, schema, tmp_path): + path = tmp_path / "existing.yaml" + path.write_text("existing configuration") + index = index_class.from_dict(schema, redis_url="rediss://[invalid") + with pytest.raises(ValueError): + index.to_yaml(path, include_connection=True) + assert path.read_text() == "existing configuration" + + +def test_yaml_overwrite_and_invalid_inputs(index_class, schema, tmp_path): + index = index_class.from_dict(schema) + path = tmp_path / "config.yaml" + index.to_yaml(path) + original = path.read_bytes() + with pytest.raises(FileExistsError): + index.to_yaml(path, overwrite=False) + assert path.read_bytes() == original + index.to_yaml(path, overwrite=True) + with pytest.raises(FileNotFoundError, match="Schema file .* does not exist"): + index_class.from_yaml(tmp_path / "missing.yaml") + path.write_text("[") + with pytest.raises(yaml.YAMLError): + index_class.from_yaml(path) + path.write_text("") + # Keep the existing schema loader's error for an empty YAML document. + with pytest.raises(AttributeError) as schema_error: + IndexSchema.from_yaml(path) + with pytest.raises(type(schema_error.value)): + index_class.from_yaml(path) + + +@pytest.mark.parametrize( + "connection", + [ + None, + [], + {"redis_url": 12}, + {"redis_url": "redis://host", "requires_override": "false"}, + {"redis_url": "redis://host", "connection_kwargs": []}, + ], +) +def test_invalid_connection_metadata(index_class, schema, connection): + with pytest.raises(ValueError, match="connection"): + index_class.from_dict({**schema, "connection": connection}) From dd3ad554e2f5e6feaff233a6dafb74a49c41e5f3 Mon Sep 17 00:00:00 2001 From: Claire-ovo <132752951+Claire-ovo@users.noreply.github.com> Date: Thu, 22 Oct 2026 23:00:15 +0800 Subject: [PATCH 2/2] fix: preserve hostless URLs and cover restore guards --- docs/api/searchindex.rst | 6 ++++-- redisvl/index/index.py | 3 +-- tests/unit/test_index_serialization.py | 15 ++++++++++++++- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/docs/api/searchindex.rst b/docs/api/searchindex.rst index 8ee5e64d..a54df8b1 100644 --- a/docs/api/searchindex.rst +++ b/docs/api/searchindex.rst @@ -18,7 +18,7 @@ Search Index Classes - Outcome of a bulk operation: how many documents matched, how many were processed, and whether the run completed. Serializing index configurations -=============================== +================================ Both index classes provide synchronous ``to_dict()`` and ``to_yaml()`` methods. They export the schema by default, without opening a connection or reading @@ -56,7 +56,9 @@ Any omission sets ``requires_override: true``. Configurations sourced from an injected client or the environment also require an override: their connection internals are not inspected. Restoring such a configuration without a nonempty ``redis_url`` or a ``redis_client`` raises ``ValueError``, even if ``REDIS_URL`` -is set. Supply the complete connection configuration at restore time: +is set. This also applies to instances created by ``from_existing()``, even +with an explicit URL, because that factory supplies a client internally. +Supply the complete connection configuration at restore time: .. code-block:: python diff --git a/redisvl/index/index.py b/redisvl/index/index.py index 706b1381..c1798429 100644 --- a/redisvl/index/index.py +++ b/redisvl/index/index.py @@ -873,6 +873,7 @@ def to_dict(self, include_connection: bool = False) -> dict[str, Any]: query parameters, fragments and non-allowlisted connection options. Any omission requires a complete explicit connection when restoring. Injected clients and environment variables are never inspected. + This includes clients supplied internally by ``from_existing()``. Args: include_connection: Include a ``connection`` mapping. Defaults to False. @@ -895,8 +896,6 @@ def to_dict(self, include_connection: bool = False) -> dict[str, Any]: if parts.scheme not in {"redis", "rediss", "unix", "redis+sentinel"}: raise ValueError endpoint = parts.netloc.rsplit("@", 1)[-1] - if parts.scheme != "unix" and not endpoint: - raise ValueError except ValueError: raise ValueError( "Cannot safely serialize the configured Redis URL" diff --git a/tests/unit/test_index_serialization.py b/tests/unit/test_index_serialization.py index b6c2a4df..67ce8708 100644 --- a/tests/unit/test_index_serialization.py +++ b/tests/unit/test_index_serialization.py @@ -71,6 +71,9 @@ def test_json_schema_roundtrip(index_class, schema): "url", [ "redis://localhost:6379/2", + "redis://", + "redis:///1", + "rediss:///2", "rediss://[2001:db8::1]:6380/3", "unix:///tmp/redis.sock", "redis+sentinel://host1:26379,host2:26380/my%2Dmaster/1", @@ -158,6 +161,16 @@ def test_incomplete_override_does_not_fall_back(index_class, schema, override): index_class.from_dict(index.to_dict(include_connection=True), **override) +@pytest.mark.parametrize("suffix", ["?db=2", "#secret"]) +def test_url_omission_alone_requires_override(index_class, schema, suffix): + index = index_class.from_dict(schema, redis_url="redis://host/0" + suffix) + config = index.to_dict(include_connection=True) + assert config["connection"]["redis_url"] == "redis://host/0" + assert config["connection"]["requires_override"] is True + with pytest.raises(ValueError, match="explicit redis_url or redis_client"): + index_class.from_dict(config) + + def test_explicit_endpoint_does_not_inherit_connection_options(index_class, schema): index = index_class.from_dict( schema, redis_url="rediss://original", connection_kwargs={"ssl": True} @@ -225,7 +238,7 @@ def test_allowlist_rejects_unsafe_values(index_class, schema, options): @pytest.mark.parametrize( "url", - ["rediss://[invalid-secret", "http://user:secret@host", "redis:///missing-host"], + ["rediss://[invalid-secret", "http://user:secret@host", "redis://[invalid-secret]"], ) def test_invalid_url_errors_do_not_expose_input(index_class, schema, url): index = index_class.from_dict(schema, redis_url=url)