Remove the CLI's hard dbt-core dependency (dbt v2 ships as dbt/dbt-oss)

.github/workflows/test-warehouse.yml

@@ -238,6 +238,15 @@ jobs:
         run: |
           docker compose up -d --build --wait spark-thrift
 
+      # The Spark image build can take most of the default 1-hour AWS session,
+      # expiring the token before the report upload steps at the end of the job.
+      - name: Refresh AWS credentials
+        if: inputs.warehouse-type == 'spark'
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
+          aws-region: eu-west-1
+
       - name: Start SQL Server
         if: inputs.warehouse-type == 'sqlserver'
         working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}

elementary/clients/dbt/dbt_installation.py

@@ -21,22 +21,26 @@ def get_dbt_core_version() -> Optional[version.Version]:
     return _get_package_version("dbt-core")
 
 
-def get_dbt_package_version() -> Optional[version.Version]:
-    """Version of the installed `dbt` package, or None if not installed.
+def get_dbt2_package_version() -> Optional[version.Version]:
+    """Version of the installed dbt v2 package, or None if not installed.
 
-    From 2.0, the `dbt` package on PyPI ships the dbt (Fusion) binary as a
-    platform wheel with no importable Python module.
+    dbt v2 (the Fusion engine) is distributed on PyPI as the `dbt` package (full
+    feature set) and the `dbt-oss` package (Apache 2 subset); both ship the dbt
+    binary. dbt-core stays on 1.x.
     """
-    return _get_package_version("dbt")
+    for package_name in ("dbt", "dbt-oss"):
+        package_version = _get_package_version(package_name)
+        if package_version is not None and package_version.major >= 2:
+            return package_version
+    return None
 
 
 def is_dbt2_binary_available() -> bool:
     env_path = os.getenv(DBT_FUSION_PATH_ENV_VAR)
     if env_path and os.path.exists(os.path.expanduser(env_path)):
         return True
 
-    dbt_package_version = get_dbt_package_version()
-    if dbt_package_version is not None and dbt_package_version.major >= 2:
+    if get_dbt2_package_version() is not None:
         return True
     return os.path.exists(os.path.expanduser(DEFAULT_DBT_FUSION_PATH))
 
@@ -49,10 +53,9 @@ def get_dbt2_binary_path() -> str:
     # When only dbt-core 1.x is installed, the `dbt` executable on PATH is its
     # entrypoint, so it can't be trusted to be the dbt 2.0 binary.
     dbt_core_version = get_dbt_core_version()
-    dbt_package_version = get_dbt_package_version()
-    dbt2_installed_via_pip = (
-        dbt_package_version is not None and dbt_package_version.major >= 2
-    ) or (dbt_core_version is not None and dbt_core_version.major >= 2)
+    dbt2_installed_via_pip = get_dbt2_package_version() is not None or (
+        dbt_core_version is not None and dbt_core_version.major >= 2
+    )
     if dbt2_installed_via_pip or dbt_core_version is None:
         which_path = shutil.which("dbt")
         if which_path:

elementary/clients/dbt/factory.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 from enum import Enum
 from typing import Any, Dict, Optional, Type
 
@@ -12,14 +13,22 @@
 )
 from elementary.clients.dbt.subprocess_dbt_runner import SubprocessDbtRunner
 from elementary.config.config import Config
+from elementary.exceptions.exceptions import NoDbtInstallationError
+from elementary.utils.log import get_logger
+
+logger = get_logger(__name__)
 
 
 class RunnerMethod(Enum):
-    # SUBPROCESS and API drive dbt-core 1.x (API requires dbt-core >= 1.5.0).
+    # DBT1_SUBPROCESS and DBT1_API drive dbt-core 1.x (DBT1_API requires
+    # dbt-core >= 1.5.0).
+    DBT1_SUBPROCESS = "subprocess"
+    DBT1_API = "api"
+    DBT2 = "dbt2"
+    # Legacy aliases for the dbt 1.x runner methods.
     SUBPROCESS = "subprocess"
     API = "api"
-    DBT2 = "dbt2"
-    # Legacy alias for DBT2 (dbt 2.0 is the Fusion engine).
+    # Legacy alias for DBT2 (dbt v2 is the Fusion engine).
     FUSION = "fusion"
 
 
@@ -74,27 +83,40 @@ def get_dbt_runner_method() -> RunnerMethod:
         return RunnerMethod(runner_method)
 
     dbt_core_version = get_dbt_core_version()
+    if is_dbt2_binary_available():
+        if dbt_core_version is not None and dbt_core_version.major < 2:
+            logger.info(
+                f"Both dbt-core {dbt_core_version} and a dbt v2 installation were "
+                "detected - using dbt v2. To use dbt-core instead, set "
+                "DBT_RUNNER_METHOD=api (or uninstall dbt v2)."
+            )
+        return RunnerMethod.DBT2
+
     if dbt_core_version is not None:
+        # dbt-core published some 2.x pre-releases before dbt v2 moved to the
+        # dbt/dbt-oss packages - drive those with the dbt v2 runner too.
         if dbt_core_version.major >= 2:
             return RunnerMethod.DBT2
         if dbt_core_version >= version.Version("1.5.0"):
-            return RunnerMethod.API
-        return RunnerMethod.SUBPROCESS
+            return RunnerMethod.DBT1_API
+        return RunnerMethod.DBT1_SUBPROCESS
 
-    if is_dbt2_binary_available():
-        return RunnerMethod.DBT2
+    # dbt may be installed without pip package metadata (e.g. a system-wide or
+    # pipx-managed dbt 1.x) - fall back to running it as a subprocess.
+    if shutil.which("dbt"):
+        return RunnerMethod.DBT1_SUBPROCESS
 
-    return RunnerMethod.SUBPROCESS
+    raise NoDbtInstallationError()
 
 
 def get_dbt_runner_class(runner_method: RunnerMethod) -> Type[CommandLineDbtRunner]:
-    if runner_method == RunnerMethod.API:
+    if runner_method == RunnerMethod.DBT1_API:
         # Import it internally since it will fail if dbt-core is not installed
         # or its version is below 1.5.0
         from elementary.clients.dbt.api_dbt_runner import APIDbtRunner
 
         return APIDbtRunner
-    elif runner_method == RunnerMethod.SUBPROCESS:
+    elif runner_method == RunnerMethod.DBT1_SUBPROCESS:
         return SubprocessDbtRunner
     elif runner_method in (RunnerMethod.DBT2, RunnerMethod.FUSION):
         return Dbt2Runner

elementary/exceptions/exceptions.py

@@ -31,6 +31,22 @@ def anonymous_tracking_context(self):
         }
 
 
+class NoDbtInstallationError(ConfigError):
+    """Exception raised when no dbt installation is found in the environment"""
+
+    def __init__(self) -> None:
+        super().__init__(
+            "No dbt installation was found. Please either install dbt v2 "
+            "(`pip install dbt` or `pip install dbt-oss`), or install "
+            "elementary-data with your warehouse's dbt 1.x adapter "
+            "(e.g. `pip install 'elementary-data[dbt1-snowflake]'`)."
+        )
+
+    @property
+    def anonymous_tracking_context(self):
+        return {"exception_message": str(self)}
+
+
 class DbtCommandError(Error):
     """Exception raised while executing a dbt command"""
 

mypy.ini

@@ -3,6 +3,10 @@ packages = elementary
 exclude = ^elementary/monitor/dbt_project/
 check_untyped_defs = true
 
+# dbt-core is an optional dependency (only installed via the dbt1-* extras)
+[mypy-dbt,dbt.*]
+ignore_missing_imports = True
+
 [mypy-alive_progress]
 ignore_missing_imports = True
 

pyproject.toml

@@ -25,9 +25,6 @@ packages = [{include = "elementary"}]
 [tool.poetry.dependencies]
 python = ">=3.10,<3.14"
 click = ">=7.0,<9.0"
-# From 2.0, dbt-core is the Fusion engine (requires Python >=3.11); adapters are
-# built into the engine, so the adapter extras below are only relevant for 1.x.
-dbt-core = ">=1.8,<3.0.0"
 requests = ">=2.28.1,<3.0.0"
 urllib3 = ">=2.8.0"  # transitive dependency via requests, which caps the major itself
 idna = ">=3.15"  # transitive dependency via requests, floored to address CVE-2026-45409 (GHSA-65pc-fj4g-8rjx) ReDoS in idna.encode()
@@ -53,6 +50,12 @@ tabulate = ">= 0.9.0"
 tenacity = ">=8.0,<10.0"
 pytz = ">= 2025.1"
 
+# There is intentionally no mandatory dbt dependency: dbt v2 ships as the `dbt` /
+# `dbt-oss` packages (dbt-core stays on 1.x) and requires no extra alongside
+# elementary-data, while dbt-core 1.x comes through the adapter extras below.
+# dbt-core is explicitly constrained in the extras so they can never resolve a
+# dbt-core 2.x release (the adapters themselves only lower-bound dbt-core).
+dbt-core = {version = ">=1.8,<2.0.0", optional = true}
 dbt-snowflake = {version = ">=1.8,<2.0.0", optional = true}
 dbt-bigquery = {version = ">=1.8,<2.0.0", optional = true}
 dbt-redshift = {version = ">=1.8,<2.0.0", optional = true}
@@ -70,24 +73,42 @@ dbt-fabric = {version = ">=1.8,<2.0.0", optional = true}
 # dbt-sqlserver 1.12 moved pyodbc (still its default backend) to the [pyodbc] extra.
 dbt-sqlserver = {version = ">=1.8,<2.0.0", optional = true, extras = ["pyodbc"]}
 [tool.poetry.extras]
-snowflake = ["dbt-snowflake"]
-bigquery = ["dbt-bigquery"]
-redshift = ["dbt-redshift"]
-postgres = ["dbt-postgres"]
-databricks = ["dbt-databricks"]
-spark = ["dbt-spark"]
-athena = ["dbt-athena-community", "pyathena"]
-clickhouse = ["dbt-clickhouse"]
-trino = ["dbt-trino"]
-duckdb = ["dbt-duckdb"]
-dremio = ["dbt-dremio"]
-fabric = ["dbt-fabric"]
-sqlserver = ["dbt-sqlserver"]
+# The extras install the dbt-core 1.x stack (adapter + dbt-core) for a warehouse.
+# They are not needed with dbt v2 (`pip install dbt` / `dbt-oss`), which has the
+# adapters built in. The dbt1-* names are the documented ones; the plain
+# warehouse names are kept as aliases for backward compatibility.
+dbt1-snowflake = ["dbt-core", "dbt-snowflake"]
+dbt1-bigquery = ["dbt-core", "dbt-bigquery"]
+dbt1-redshift = ["dbt-core", "dbt-redshift"]
+dbt1-postgres = ["dbt-core", "dbt-postgres"]
+dbt1-databricks = ["dbt-core", "dbt-databricks"]
+dbt1-spark = ["dbt-core", "dbt-spark"]
+dbt1-athena = ["dbt-core", "dbt-athena-community", "pyathena"]
+dbt1-clickhouse = ["dbt-core", "dbt-clickhouse"]
+dbt1-trino = ["dbt-core", "dbt-trino"]
+dbt1-duckdb = ["dbt-core", "dbt-duckdb"]
+dbt1-dremio = ["dbt-core", "dbt-dremio"]
+dbt1-fabric = ["dbt-core", "dbt-fabric"]
+dbt1-sqlserver = ["dbt-core", "dbt-sqlserver"]
+snowflake = ["dbt-core", "dbt-snowflake"]
+bigquery = ["dbt-core", "dbt-bigquery"]
+redshift = ["dbt-core", "dbt-redshift"]
+postgres = ["dbt-core", "dbt-postgres"]
+databricks = ["dbt-core", "dbt-databricks"]
+spark = ["dbt-core", "dbt-spark"]
+athena = ["dbt-core", "dbt-athena-community", "pyathena"]
+clickhouse = ["dbt-core", "dbt-clickhouse"]
+trino = ["dbt-core", "dbt-trino"]
+duckdb = ["dbt-core", "dbt-duckdb"]
+dremio = ["dbt-core", "dbt-dremio"]
+fabric = ["dbt-core", "dbt-fabric"]
+sqlserver = ["dbt-core", "dbt-sqlserver"]
 # dbt-fabricspark and dbt-vertica are removed from the lockfile resolution because they pin
 # outdated transitive dependencies (dbt-core==1.8.5, azure-cli pre-release) that block
 # security patches for deepdiff, protobuf, and pyopenssl. Users who need these adapters
 # can still install them directly (e.g. pip install dbt-fabricspark dbt-vertica).
-all = ["dbt-snowflake", "dbt-bigquery", "dbt-redshift", "dbt-postgres", "dbt-databricks", "dbt-spark", "dbt-athena-community", "pyathena", "dbt-trino", "dbt-clickhouse", "dbt-duckdb", "dbt-dremio", "dbt-fabric", "dbt-sqlserver"]
+dbt1-all = ["dbt-core", "dbt-snowflake", "dbt-bigquery", "dbt-redshift", "dbt-postgres", "dbt-databricks", "dbt-spark", "dbt-athena-community", "pyathena", "dbt-trino", "dbt-clickhouse", "dbt-duckdb", "dbt-dremio", "dbt-fabric", "dbt-sqlserver"]
+all = ["dbt-core", "dbt-snowflake", "dbt-bigquery", "dbt-redshift", "dbt-postgres", "dbt-databricks", "dbt-spark", "dbt-athena-community", "pyathena", "dbt-trino", "dbt-clickhouse", "dbt-duckdb", "dbt-dremio", "dbt-fabric", "dbt-sqlserver"]
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]