From 5eb68df33daf7b57b5c35274e2f5cad87449b08b Mon Sep 17 00:00:00 2001 From: Anton Dziatkovskii <194927794+tonydzi@users.noreply.github.com> Date: Tue, 1 Sep 2026 08:32:05 -0700 Subject: [PATCH] Python: document connector extras and optional dependency compatibility Adds docs/CONNECTOR_EXTRAS.md, a table mapping every public Python connector to its install extra and the upstream packages and constraints that extra declares, plus a section on how those constraints are validated in CI. Adds tests/unit/test_connector_extras_doc.py so the table cannot drift: it parses the doc and pyproject.toml and fails on an unknown extra, an undocumented connector extra, a package-name or version mismatch, an import path that does not exist, or a row claiming a package its own connector never imports. Stdlib plus pytest only, no network, no project extras. pyproject.toml and uv.lock are unchanged: no range was widened and no connector moved into the default install. Closes #14327 Assisted-by: Claude Code/claude-opus-5[1m] Machine: MacBook-Anton Account: a Operator: robot:connector-butcher-daily Signed-off-by: tonydzi <194927794+tonydzi@users.noreply.github.com> --- python/README.md | 2 + python/docs/CONNECTOR_EXTRAS.md | 142 +++++++++++ .../tests/unit/test_connector_extras_doc.py | 239 ++++++++++++++++++ 3 files changed, 383 insertions(+) create mode 100644 python/docs/CONNECTOR_EXTRAS.md create mode 100644 python/tests/unit/test_connector_extras_doc.py diff --git a/python/README.md b/python/README.md index 6dd080b09e7f..0eb6f7de62ea 100644 --- a/python/README.md +++ b/python/README.md @@ -23,6 +23,8 @@ pip install --upgrade semantic-kernel[hugging_face] pip install --upgrade semantic-kernel[all] ``` +Every connector and the extra it needs is listed in [Connector extras and optional dependencies](./docs/CONNECTOR_EXTRAS.md). + Supported Platforms: - Python: 3.10+ - OS: Windows, macOS, Linux diff --git a/python/docs/CONNECTOR_EXTRAS.md b/python/docs/CONNECTOR_EXTRAS.md new file mode 100644 index 000000000000..776e70cddf55 --- /dev/null +++ b/python/docs/CONNECTOR_EXTRAS.md @@ -0,0 +1,142 @@ +# Python Connector Extras and Optional Dependencies + +This page maps every public Python connector under `semantic_kernel/connectors/` to the +install extra it needs and to the upstream package(s) and version constraints that extra +declares in [`pyproject.toml`](../pyproject.toml). + +Connectors that need no extra say `none` explicitly — they only rely on packages that are +already part of the base `semantic-kernel` install. + +## How to install + +```bash +# from PyPI +pip install semantic-kernel[] +pip install semantic-kernel[azure,redis] # more than one extra + +# from a clone of this repository +uv sync --extra +uv sync --all-extras --dev # everything, what CI installs +``` + +Extra names are the keys of `[project.optional-dependencies]` in +[`pyproject.toml`](../pyproject.toml). Installing an extra never changes the base install: +these packages are optional on purpose, and adding a connector to the default install or +widening a constraint is out of scope for this page. + +## AI service connectors + +Import path is the public module; see [AI connectors README](../semantic_kernel/connectors/ai/README.md) +for the concrete service classes each one exports. + +| Connector | Import path | Install extra | Upstream package(s) & constraint | +| --- | --- | --- | --- | +| Anthropic | `semantic_kernel.connectors.ai.anthropic` | `anthropic` | `anthropic ~= 0.32` | +| Azure AI Inference | `semantic_kernel.connectors.ai.azure_ai_inference` | `azure` | `azure-ai-inference >= 1.0.0b6`, `azure-core-tracing-opentelemetry >= 1.0.0b11` | +| Amazon Bedrock | `semantic_kernel.connectors.ai.bedrock` | `aws` | `boto3>=1.36.4,<1.43.0` | +| Google AI / Vertex AI | `semantic_kernel.connectors.ai.google` | `google` | `google-genai >= 1.51,< 1.75`, `google-cloud-aiplatform>=1.114,<1.134` | +| Hugging Face | `semantic_kernel.connectors.ai.hugging_face` | `hugging_face` | `transformers[torch] >= 4.28,< 6.0`, `sentence-transformers >= 2.2,< 6.0`, `torch==2.13.0` | +| Mistral AI | `semantic_kernel.connectors.ai.mistral_ai` | `mistralai` | `mistralai >= 1.2,< 2.7.3` | +| NVIDIA NIM | `semantic_kernel.connectors.ai.nvidia` | `none` | `openai >= 2.0.0` (base dependency; the connector talks to NIM through the OpenAI client) | +| Ollama | `semantic_kernel.connectors.ai.ollama` | `ollama` | `ollama ~= 0.4` | +| ONNX GenAI | `semantic_kernel.connectors.ai.onnx` | `onnx` | `onnxruntime==1.22.1`, `onnxruntime>=1.26.0`, `onnxruntime-genai==0.9.0`, `onnxruntime-genai==0.14.1` (selected by python_version marker) | +| OpenAI / Azure OpenAI | `semantic_kernel.connectors.ai.open_ai` | `none` | `openai >= 2.0.0` (base dependency) | + +The realtime clients exported by `semantic_kernel.connectors.ai.open_ai` +(`OpenAIRealtimeWebsocket`, `AzureRealtimeWebRTC`, ...) need `websockets` and `aiortc`. +Both are base dependencies, so no extra is required; the `realtime` extra declares the same +two packages and exists as an explicit opt-in name rather than as an additional install. + +## Vector store, search and protocol connectors + +| Connector | Import path | Install extra | Upstream package(s) & constraint | +| --- | --- | --- | --- | +| Azure AI Search | `semantic_kernel.connectors.azure_ai_search` | `azure` | `azure-search-documents >= 11.6.0b4, < 13.0.0` | +| Azure Cosmos DB NoSQL | `semantic_kernel.connectors.azure_cosmos_db` | `azure` | `azure-cosmos ~= 4.7` | +| Azure Cosmos DB for MongoDB (vCore) | `semantic_kernel.connectors.azure_cosmos_db` | `azure` and `mongo` | `azure-cosmos ~= 4.7`, `pymongo >= 4.8.0, < 4.17` | +| Brave Search | `semantic_kernel.connectors.brave` | `none` | `httpx`, already installed as a transitive dependency of the base install | +| Chroma | `semantic_kernel.connectors.chroma` | `chroma` | `chromadb >= 0.5,< 1.6` | +| Faiss | `semantic_kernel.connectors.faiss` | `faiss` | `faiss-cpu>=1.10.0` | +| Google Web Search | `semantic_kernel.connectors.google_search` | `none` | `httpx`, already installed as a transitive dependency of the base install | +| In-memory store | `semantic_kernel.connectors.in_memory` | `none` | `numpy >= 1.25.0`, `numpy >= 1.26.0`, `scipy>=1.15.1` (base dependencies) | +| MCP (stdio, SSE, streamable HTTP, websocket) | `semantic_kernel.connectors.mcp` | `mcp` (optional — `mcp` is also a base dependency) | `mcp>=1.8,<2.0` | +| MongoDB Atlas | `semantic_kernel.connectors.mongodb` | `mongo` | `pymongo >= 4.8.0, < 4.17`, `motor >= 3.3.2,< 3.8.0` | +| OpenAPI plugin | `semantic_kernel.connectors.openapi_plugin` | `none` | `openapi_core >= 0.18,<0.20`, `prance >= 23.6.21,< 26.7.20` (base dependencies) | +| Oracle 23ai | `semantic_kernel.connectors.oracle` | `oracledb` | `oracledb >= 3.4.1` | +| Pinecone | `semantic_kernel.connectors.pinecone` | `pinecone` | `pinecone[asyncio, grpc] ~= 7.0`, `pinecone[asyncio, grpc] ~= 7.3` (selected by sys_platform marker) | +| Postgres / pgvector | `semantic_kernel.connectors.postgres` | `postgres` | `psycopg[binary,pool] ~= 3.2` | +| Qdrant | `semantic_kernel.connectors.qdrant` | `qdrant` | `qdrant-client ~= 1.9` | +| Redis | `semantic_kernel.connectors.redis` | `redis` | `redis[hiredis] >= 6,< 8`, `redisvl ~= 0.4`, `types-redis ~= 4.6.0.20240425` | +| SQL Server | `semantic_kernel.connectors.sql_server` | `sql` | `pyodbc >= 5.2` | +| Weaviate | `semantic_kernel.connectors.weaviate` | `weaviate` | `weaviate-client>=4.17.0,<5.0` | + +## Deprecated memory-store connectors + +Everything under `semantic_kernel/connectors/memory_stores/` is marked `@deprecated` and +will be removed in a future release; prefer the vector-store connectors in the table above. +They are listed here because some extras exist only for them. + +| Connector | Import path | Install extra | Upstream package(s) & constraint | +| --- | --- | --- | --- | +| AstraDB (deprecated) | `semantic_kernel.connectors.memory_stores.astradb` | `none` | `aiohttp ~= 3.8` (base dependency) | +| Azure Cognitive Search (deprecated) | `semantic_kernel.connectors.memory_stores.azure_cognitive_search` | `azure` | `azure-search-documents >= 11.6.0b4, < 13.0.0` | +| Azure Cosmos DB Mongo vCore (deprecated) | `semantic_kernel.connectors.memory_stores.azure_cosmosdb` | `mongo` | `pymongo >= 4.8.0, < 4.17` | +| Azure Cosmos DB NoSQL (deprecated) | `semantic_kernel.connectors.memory_stores.azure_cosmosdb_no_sql` | `azure` | `azure-cosmos ~= 4.7` | +| Chroma (deprecated) | `semantic_kernel.connectors.memory_stores.chroma` | `chroma` | `chromadb >= 0.5,< 1.6` | +| Milvus (deprecated) | `semantic_kernel.connectors.memory_stores.milvus` | `milvus` | `pymilvus >= 2.3,< 2.7`, `milvus >= 2.3,<2.3.8` (not installed on Windows) | +| MongoDB Atlas (deprecated) | `semantic_kernel.connectors.memory_stores.mongodb_atlas` | `mongo` | `pymongo >= 4.8.0, < 4.17` | +| Pinecone (deprecated) | `semantic_kernel.connectors.memory_stores.pinecone` | `pinecone` | `pinecone[asyncio, grpc] ~= 7.0`, `pinecone[asyncio, grpc] ~= 7.3` | +| Postgres (deprecated) | `semantic_kernel.connectors.memory_stores.postgres` | `postgres` | `psycopg[binary,pool] ~= 3.2` | +| Qdrant (deprecated) | `semantic_kernel.connectors.memory_stores.qdrant` | `qdrant` | `qdrant-client ~= 1.9` | +| Redis (deprecated) | `semantic_kernel.connectors.memory_stores.redis` | `redis` | `redis[hiredis] >= 6,< 8` | +| USearch (deprecated) | `semantic_kernel.connectors.memory_stores.usearch` | `usearch` | `usearch >= 2.16,< 2.25`, `pyarrow>=12.0,<24.0` | +| Weaviate (deprecated) | `semantic_kernel.connectors.memory_stores.weaviate` | `weaviate` | `weaviate-client>=4.17.0,<5.0` | + +## Extras that are not connectors + +These extras exist in `pyproject.toml` but do not belong to a connector, so they have no row +above: `autogen` and `copilotstudio` (agent integrations under `semantic_kernel/agents/`), +`notebooks` and `pandas` (sample/tooling helpers), and `realtime` (see the note under the AI +service table). + +## Connector-specific configuration docs + +- [All settings — constructor arguments, environment variables and settings classes per connector](../samples/concepts/setup/ALL_SETTINGS.md) +- [AI connectors overview](../semantic_kernel/connectors/ai/README.md) +- [Amazon Bedrock connector](../semantic_kernel/connectors/ai/bedrock/README.md) +- [Google AI / Vertex AI connector](../semantic_kernel/connectors/ai/google/README.md) +- [NVIDIA NIM connector](../semantic_kernel/connectors/ai/nvidia/README.md) +- [MongoDB Atlas memory store](../semantic_kernel/connectors/memory_stores/mongodb_atlas/README.md) +- [Redis memory store](../semantic_kernel/connectors/memory_stores/redis/README.md) +- [Weaviate memory store](../semantic_kernel/connectors/memory_stores/weaviate/README.md) +- [Dev setup — installing extras from a clone](../DEV_SETUP.md) + +## How version constraints are validated in CI + +Constraints live in exactly one place, `python/pyproject.toml`. `python/uv.lock` holds the +resolved versions for the three platforms declared in `[tool.uv] environments` +(`darwin`, `linux`, `win32`). The workflows below are what actually exercise them: + +| Workflow | Install step | What it proves | +| --- | --- | --- | +| `.github/workflows/python-unit-tests.yml` | `uv sync --all-extras --dev -U --prerelease=if-necessary-or-explicit` | Every extra resolves together, on Python 3.10 / 3.11 / 3.12 across ubuntu, windows and macos (plus an experimental 3.13 ubuntu job). `-U` re-resolves to the newest versions the constraints allow, so a constraint that no longer resolves fails the PR. Tests then run with `uv run --frozen pytest ./tests/unit`. | +| `.github/workflows/python-test-coverage.yml` | `uv sync --all-extras --dev -U --prerelease=if-necessary-or-explicit` | Same all-extras resolution on Python 3.10, then the unit suite with coverage. | +| `.github/workflows/python-lint.yml` | `uv sync --all-extras --dev` | Installs from the committed `uv.lock` (re-resolving only when `pyproject.toml` changed) on Python 3.10, then runs pre-commit and `uv run mypy -p semantic_kernel` against the installed optional packages. | +| `.github/workflows/python-integration-tests.yml` | `uv sync --all-extras --dev` | The integration jobs install every extra from the committed `uv.lock` before talking to the real services. | + +`astral-sh/setup-uv` is cached on `cache-dependency-glob: "**/uv.lock"` in each of those +workflows, so a lock change invalidates the cache and forces a fresh install. + +Two limits worth stating plainly: + +- CI installs **all** extras together. It does not install a single extra on its own, so + "extra `X` alone is enough to import connector `X`" is not verified by CI today. The one + exception is the dapr job in `python-unit-tests.yml`, which installs only `--extra pandas`. +- This page itself is kept in sync by + [`tests/unit/test_connector_extras_doc.py`](../tests/unit/test_connector_extras_doc.py), + which runs in the unit-test and coverage workflows above. It parses the tables here and + `pyproject.toml` and fails when an extra is renamed, added or removed, or when the upstream + package names or version constraints listed here stop matching `pyproject.toml`. Version + specifiers in a `none` row are checked against the base `[project] dependencies`; a package + named without a version specifier (such as `httpx` above) is documentation only and is not + machine-checked. diff --git a/python/tests/unit/test_connector_extras_doc.py b/python/tests/unit/test_connector_extras_doc.py new file mode 100644 index 000000000000..570b7e982f5c --- /dev/null +++ b/python/tests/unit/test_connector_extras_doc.py @@ -0,0 +1,239 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Keeps docs/CONNECTOR_EXTRAS.md checkable against the package metadata in pyproject.toml. + +The doc lists, for every public connector, the install extra it needs and the upstream +packages that extra declares. This test parses both files and fails when they drift: + +* an extra named in the doc that does not exist in pyproject.toml, +* a connector extra in pyproject.toml that no doc row mentions, +* upstream package names for an extra that differ between the two files, +* a version specifier in the doc that no longer matches pyproject.toml, +* an import path in the doc that does not exist on disk, +* a row claiming a package that its own connector never imports. + +Stdlib only, no network, no project extras required. +""" + +import re +import sys +from pathlib import Path + +if sys.version_info >= (3, 11): + import tomllib +else: # pragma: no cover - Python 3.10 only, tomli ships with pytest there + import tomli as tomllib + +PYTHON_ROOT = Path(__file__).resolve().parents[2] +PYPROJECT = PYTHON_ROOT / "pyproject.toml" +DOC = PYTHON_ROOT / "docs" / "CONNECTOR_EXTRAS.md" + +# Table header that marks a connector table in the doc. Other tables in the doc are ignored. +TABLE_HEADER = ("Connector", "Import path", "Install extra", "Upstream package(s) & constraint") + +# Extras that exist in pyproject.toml but are deliberately not connector extras, so they have +# no row in the doc. Listed explicitly so that intent is stated instead of silently ignored. +NON_CONNECTOR_EXTRAS = { + "autogen", # agent integration, semantic_kernel/agents/autogen + "copilotstudio", # agent integration, semantic_kernel/agents/copilot_studio + "notebooks", # ipykernel, for the sample notebooks + "pandas", # sample and test helper + "realtime", # websockets/aiortc, both already base dependencies +} + +BACKTICKED = re.compile(r"`([^`]+)`") +DIST_NAME = re.compile(r"^([A-Za-z0-9][A-Za-z0-9._-]*)") +HAS_SPECIFIER = re.compile(r"[<>=!~]") +EXTRA_NAME = re.compile(r"^[a-z][a-z0-9_]*$") +IMPORTED = re.compile(r"^\s*(?:from|import)\s+([A-Za-z_][\w.]*)", re.MULTILINE) + + +def _canonical_name(requirement: str) -> str: + """Return the PEP 503 normalized distribution name of a requirement string.""" + match = DIST_NAME.match(requirement.strip()) + if not match: + return "" + return re.sub(r"[-_.]+", "-", match.group(1)).lower() + + +def _normalize(requirement: str) -> str: + """Drop the environment marker and all whitespace so two spellings compare equal.""" + return re.sub(r"\s+", "", requirement.split(";", 1)[0]) + + +def _load_pyproject() -> tuple[list[str], dict[str, list[str]]]: + with PYPROJECT.open("rb") as handle: + data = tomllib.load(handle) + project = data["project"] + return project["dependencies"], project["optional-dependencies"] + + +def _parse_rows() -> list[dict]: + """Parse every connector table in the doc into rows.""" + rows: list[dict] = [] + lines = DOC.read_text(encoding="utf-8").splitlines() + index = 0 + while index < len(lines): + cells = _split_row(lines[index]) + if tuple(cells) != TABLE_HEADER: + index += 1 + continue + index += 2 # skip the header and the `| --- |` separator + while index < len(lines) and lines[index].startswith("|"): + cells = _split_row(lines[index]) + assert len(cells) == 4, f"{DOC.name} line {index + 1}: expected 4 columns, got {len(cells)}" + rows.append({ + "line": index + 1, + "connector": cells[0], + "import_path": _single_backticked(cells[1], index + 1), + "extras": _extras_of(cells[2], index + 1), + "upstream": BACKTICKED.findall(cells[3]), + }) + index += 1 + return rows + + +def _split_row(line: str) -> list[str]: + if not line.startswith("|"): + return [] + return [cell.strip() for cell in line.strip().strip("|").split("|")] + + +def _single_backticked(cell: str, line_no: int) -> str: + found = BACKTICKED.findall(cell) + assert len(found) == 1, f"{DOC.name} line {line_no}: expected exactly one `import path`, got {found}" + return found[0] + + +def _extras_of(cell: str, line_no: int) -> set[str]: + found = {token for token in BACKTICKED.findall(cell) if EXTRA_NAME.match(token)} + assert found, f"{DOC.name} line {line_no}: install extra column must name an extra or `none`" + return found + + +def test_doc_and_pyproject_exist(): + assert DOC.is_file(), f"{DOC} is missing" + assert PYPROJECT.is_file(), f"{PYPROJECT} is missing" + assert _parse_rows(), "no connector table rows were parsed from the doc" + + +def test_non_connector_allow_list_is_real(): + """The allow-list must only name extras that actually exist, so it cannot go stale.""" + _, optional = _load_pyproject() + unknown = NON_CONNECTOR_EXTRAS - set(optional) + assert not unknown, f"NON_CONNECTOR_EXTRAS names extras that no longer exist in pyproject.toml: {sorted(unknown)}" + + +def test_every_documented_extra_exists_in_pyproject(): + _, optional = _load_pyproject() + for row in _parse_rows(): + for extra in sorted(row["extras"]): + if extra == "none": + continue + assert extra in optional, ( + f"{DOC.name} line {row['line']} ({row['connector']}): extra '{extra}' is not declared in " + f"[project.optional-dependencies]" + ) + + +def test_every_connector_extra_is_documented(): + _, optional = _load_pyproject() + documented = {extra for row in _parse_rows() for extra in row["extras"]} - {"none"} + missing = set(optional) - NON_CONNECTOR_EXTRAS - documented + assert not missing, ( + f"these extras are declared in pyproject.toml but have no row in {DOC.name}: {sorted(missing)}. " + f"Add a row, or add the extra to NON_CONNECTOR_EXTRAS with a reason." + ) + + +def test_upstream_package_names_match_pyproject(): + """The packages listed per extra in the doc must be exactly what that extra declares.""" + _, optional = _load_pyproject() + documented: dict[str, set[str]] = {} + for row in _parse_rows(): + extras = row["extras"] - {"none"} + if not extras: + continue + for requirement in row["upstream"]: + name = _canonical_name(requirement) + owners = [extra for extra in extras if name in {_canonical_name(r) for r in optional[extra]}] + assert owners, ( + f"{DOC.name} line {row['line']} ({row['connector']}): '{requirement}' is not declared by any of " + f"{sorted(extras)} in pyproject.toml" + ) + for owner in owners: + documented.setdefault(owner, set()).add(name) + + for extra, names in sorted(documented.items()): + expected = {_canonical_name(requirement) for requirement in optional[extra]} + assert names == expected, ( + f"extra '{extra}': {DOC.name} documents {sorted(names)} but pyproject.toml declares {sorted(expected)}" + ) + + +def test_version_constraints_match_pyproject(): + """Every version specifier printed in the doc must still be the one pyproject.toml declares.""" + dependencies, optional = _load_pyproject() + base = {_normalize(requirement) for requirement in dependencies} + for row in _parse_rows(): + extras = row["extras"] - {"none"} + allowed = base if not extras else {_normalize(r) for extra in extras for r in optional[extra]} + for requirement in row["upstream"]: + if not HAS_SPECIFIER.search(requirement): + continue # documentation-only entry, e.g. a transitive dependency + assert _normalize(requirement) in allowed, ( + f"{DOC.name} line {row['line']} ({row['connector']}): '{requirement}' does not match the " + f"constraint declared in pyproject.toml for {sorted(extras) or '[project] dependencies'}" + ) + + +def test_documented_import_paths_exist(): + for row in _parse_rows(): + parts = row["import_path"].split(".") + package = PYTHON_ROOT.joinpath(*parts) + module = package.with_suffix(".py") + assert (package / "__init__.py").is_file() or module.is_file(), ( + f"{DOC.name} line {row['line']} ({row['connector']}): import path '{row['import_path']}' " + f"does not exist under {PYTHON_ROOT}" + ) + + +def _import_names(import_path: str) -> set[str]: + """Every module name imported by the connector module or package at `import_path`.""" + parts = import_path.split(".") + package = PYTHON_ROOT.joinpath(*parts) + sources = sorted(package.rglob("*.py")) if package.is_dir() else [package.with_suffix(".py")] + names: set[str] = set() + for source in sources: + for match in IMPORTED.finditer(source.read_text(encoding="utf-8")): + names.add(match.group(1)) + return names + + +def _import_candidates(requirement: str) -> set[str]: + """Plausible import names for a distribution name, widest last.""" + name = _canonical_name(requirement) + return {name.replace("-", "_"), name.replace("-", "."), name.split("-", 1)[0]} + + +def test_documented_packages_are_actually_imported_by_the_connector(): + """A row must not claim a package its own connector never imports. + + This is what stops a self-consistent wrong pairing: without it a row can name a real + extra whose real packages simply belong to a different connector, and every other + check in this file still passes. + """ + for row in _parse_rows(): + if not row["extras"] - {"none"}: + continue # base-install connectors, nothing extra to attribute + imported = _import_names(row["import_path"]) + hit = any( + candidate == name or name.startswith(f"{candidate}.") + for requirement in row["upstream"] + for candidate in _import_candidates(requirement) + for name in imported + ) + assert hit, ( + f"{DOC.name} line {row['line']} ({row['connector']}): none of {row['upstream']} is imported " + f"anywhere under '{row['import_path']}' -- the row attributes a package to the wrong connector" + )